Skip to content

SRE-3584 Fix test reporting #17568

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
JohnMalmberg wants to merge 3 commits into
base: master
Choose a base branch
from
+47 −26
Open

SRE-3584 Fix test reporting #17568

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e826c46
SRE-3584 Fix test reporting
JohnMalmberg Feb 17, 2026
86c3b29
Merge branch 'master' into malmberg/sre-3584
JohnMalmberg Feb 23, 2026
9759306
sre-3584 Updates per review comments.
JohnMalmberg Feb 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -886,7 +886,8 @@ pipeline {
job_step_update(
functionalTest(
inst_repos: daosRepos(),
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + ' mercury-libfabric',
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') +
' mercury-libfabric',
test_function: 'runTestFunctionalV2'))
}
post {
Expand All @@ -908,7 +909,8 @@ pipeline {
job_step_update(
functionalTest(
inst_repos: daosRepos(),
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + ' mercury-libfabric',
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') +
' mercury-libfabric',
test_function: 'runTestFunctionalV2'))
}
post {
Expand All @@ -930,7 +932,8 @@ pipeline {
job_step_update(
functionalTest(
inst_repos: daosRepos(),
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + ' mercury-libfabric',
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') +
' mercury-libfabric',
test_function: 'runTestFunctionalV2'))
}
post {
Expand All @@ -952,7 +955,8 @@ pipeline {
job_step_update(
functionalTest(
inst_repos: daosRepos(),
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + ' mercury-libfabric',
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') +
' mercury-libfabric',
test_function: 'runTestFunctionalV2',
image_version: 'leap15.6'))
}
Expand All @@ -975,7 +979,8 @@ pipeline {
job_step_update(
functionalTest(
inst_repos: daosRepos(),
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') + ' mercury-libfabric',
inst_rpms: functionalPackages(1, next_version(), 'tests-internal') +
' mercury-libfabric',
test_function: 'runTestFunctionalV2'))
}
post {
Expand Down
38 changes: 24 additions & 14 deletions ci/functional/test_main.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/bin/bash
#
# Copyright 2020-2024 Intel Corporation.
# Copyright 2025 Hewlett Packard Enterprise Development LP
# Copyright 2025-2026 Hewlett Packard Enterprise Development LP
#
# SPDX-License-Identifier: BSD-2-Clause-Patent
#
Expand All @@ -14,6 +14,13 @@ fi

test_tag="$TEST_TAG"

: "${NODELIST:=localhost}"
: "${TEST_RPMS:=false}"
: "${STAGE_NAME:=unknown}"

def_node_count="$(nodeset -c "$NODELIST")"
: "${NODE_COUNT:=$def_node_count}"

tnodes=$(echo "$NODELIST" | cut -d ',' -f 1-"$NODE_COUNT")
first_node=${NODELIST%%,*}

Expand Down Expand Up @@ -42,14 +49,17 @@ cluster_reboot () {
test_cluster() {
# Test that all nodes in the cluster are healthy
clush -B -S -o '-i ci_key' -l root -w "${tnodes}" \
"OPERATIONS_EMAIL=${OPERATIONS_EMAIL} \
"OPERATIONS_EMAIL=${OPERATIONS_EMAIL:-} \
FIRST_NODE=${first_node} \
TEST_RPMS=${TEST_RPMS} \
NODELIST=${tnodes} \
BUILD_URL=\"${BUILD_URL:-Unknown in GHA}\" \
STAGE_NAME=\"$STAGE_NAME\" \
STAGE_NAME=\"${STAGE_NAME}\" \
JENKINS_URL=\"${JENKINS_URL:-}\" \
DAOS_DEVOPS_EMAIL=\"${DAOS_DEVOPS_EMAIL:-}\" \
DAOS_INFINIBAND=${DAOS_INFINIBAND:-} \
DAOS_NVME=${DAOS_NVME:-} \
DAOS_PMEM=${DAOS_PMEM:-} \
$(cat ci/functional/test_main_prep_node.sh)"
}

Expand All @@ -65,7 +75,7 @@ if ! test_cluster; then
echo "Hardware test failed again after reboot"
fi
else
echo "Cluster reboot failed"
echo "Cluster reboot failed"
fi
else
hardware_ok=true
Expand All @@ -88,6 +98,7 @@ trap 'clush -B -S -o "-i ci_key" -l root -w "${tnodes}" '\

# Setup the Jenkins build artifacts directory before running the tests to ensure
# there is enough disk space to report the results.
# Even though STAGE_NAME forced to be set, shellcheck wants this syntax.
rm -rf "${STAGE_NAME:?ERROR: STAGE_NAME is not defined}/"
mkdir "${STAGE_NAME:?ERROR: STAGE_NAME is not defined}/"

Expand All @@ -98,24 +109,23 @@ rm -rf install/lib/daos/TESTING/ftest/avocado ./*_results.xml
mkdir -p install/lib/daos/TESTING/ftest/avocado/job-results

if "$hardware_ok"; then
if $TEST_RPMS; then
if "$TEST_RPMS"; then
# shellcheck disable=SC2029
ssh -i ci_key -l jenkins "${first_node}" \
"TEST_TAG=\"$test_tag\" \
TNODES=\"$tnodes\" \
FTEST_ARG=\"${FTEST_ARG:-}\" \
WITH_VALGRIND=\"${WITH_VALGRIND:-}\" \
STAGE_NAME=\"$STAGE_NAME\" \
DAOS_HTTPS_PROXY=\"${DAOS_HTTPS_PROXY:-}\" \
ssh -i ci_key -l jenkins "${first_node}" \
"TEST_TAG=\"$test_tag\" \
TNODES=\"$tnodes\" \
FTEST_ARG=\"${FTEST_ARG:-}\" \
WITH_VALGRIND=\"${WITH_VALGRIND:-}\" \
STAGE_NAME=\"${STAGE_NAME}\" \
DAOS_HTTPS_PROXY=\"${DAOS_HTTPS_PROXY:-}\" \
$(cat ci/functional/test_main_node.sh)"
else
./ftest.sh "$test_tag" "$tnodes" "$FTEST_ARG"
./ftest.sh "$test_tag" "$tnodes" "${FTEST_ARG:-}"
fi
fi

# Now rename the previously collected hardware test data for Jenkins
# to use them for Junit processing.
: "${STAGE_NAME:=}"
mkdir -p "${STAGE_NAME}/hardware_prep/"
for node in ${tnodes//,/ }; do
old_name="./hardware_prep_node_results.xml.$node"
Expand Down
20 changes: 13 additions & 7 deletions ci/functional/test_main_prep_node.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,12 @@ mail_domain="${domain1%%/*}"
: "${DAOS_PMEM:=0}"
: "${DAOS_NVME:=0}"

#cn is for a cleaned up stage name.
cn=$(echo "$STAGE_NAME" | sed 's/[^a-zA-Z0-9_]/_/g' | sed 's/__*/_/g')

result=0
mail_message=''
mail_type='warning'
nl="
"

Expand Down Expand Up @@ -51,7 +55,7 @@ function do_mail {
fi
# shellcheck disable=SC2059
build_info="BUILD_URL = $BUILD_URL$nl STAGE = $STAGE_NAME$nl$nl"
mail -s "Hardware check failed after reboot!" \
mail -s "Hardware check $mail_type after reboot!" \
-r "$DAOS_DEVOPS_EMAIL" "$OPERATIONS_EMAIL" \
<<< "$build_info$mail_message"
set -x
Expand Down Expand Up @@ -340,21 +344,23 @@ if [ -n "$FIRST_NODE" ] && ! grep /mnt/share /proc/mounts; then
mount "$FIRST_NODE":/export/share /mnt/share
fi

# Defaulting the package to "(root)" for now as then Jenkins
# will default to setting putting the outer stage name and
# inner stage name in the full test name.
ts="Hardware"
# The package name defaults to "(root)" unless there is a dot in the
# testsuite name, in which case the package name is the part before
# the last dot in the testsuite name.
pn="Hardware"
tf="failures=\"$testfails\""
te="errors=\"0\""
tc="tests=\"$testruns\""

# shellcheck disable=SC2089
junit_xml="<testsuite name=\"$ts\" skipped=\"0\" $tf $te $tc>$nl
junit_xml="<testsuite name=\"${pn}.${cn}\" skipped=\"0\" $tf $te $tc>$nl
$testcases</testsuite>$nl"

# Each junit file needs the same name for when they are collected.
echo "$junit_xml" > "./hardware_prep_node_results.xml"

if [ "$testfails" -gt 0 ]; then
mail_type='failed'
fi
do_mail

if [ "$result" -ne 0 ]; then
Expand Down
Loading