Add infrastructure for HIP catch tests; add unit/compiler tests #297
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
ritter-x2a
reviewed
Dec 15, 2025
Comment on lines
+573
to
+756
| # Generate enhanced summary script with individual test tracking | ||
| set(_summary_script "${CMAKE_CURRENT_BINARY_DIR}/catch_tests/summary_${CATEGORY}_${SUBDIR}_${VARIANT_SUFFIX}.sh") | ||
| file(WRITE "${_summary_script}" "#!/bin/bash\n") | ||
| file(APPEND "${_summary_script}" "# Enhanced summary script for Catch2 TEST_CASE statistics\n") | ||
| file(APPEND "${_summary_script}" "cd ${CMAKE_CURRENT_BINARY_DIR}\n") | ||
| file(APPEND "${_summary_script}" "echo \"\"\n") | ||
| file(APPEND "${_summary_script}" "echo \"========================================\"\n") | ||
| file(APPEND "${_summary_script}" "echo \"Detailed Test Summary:\"\n") | ||
| file(APPEND "${_summary_script}" "TOTAL_FILES=0\n") | ||
| file(APPEND "${_summary_script}" "TOTAL_TESTS=0\n") | ||
| file(APPEND "${_summary_script}" "PASSED_TESTS=0\n") | ||
| file(APPEND "${_summary_script}" "FAILED_TESTS=0\n") | ||
| file(APPEND "${_summary_script}" "SKIPPED_TESTS=0\n") | ||
| file(APPEND "${_summary_script}" "CRASHED_TESTS=0\n") | ||
| file(APPEND "${_summary_script}" "# Arrays to track test names for triage\n") | ||
| file(APPEND "${_summary_script}" "FAILED_LIST=\"\"\n") | ||
| file(APPEND "${_summary_script}" "SKIPPED_LIST=\"\"\n") | ||
| file(APPEND "${_summary_script}" "CRASHED_LIST=\"\"\n") | ||
| file(APPEND "${_summary_script}" "for test in catch_tests/catch_${CATEGORY}_${SUBDIR}_*-${VARIANT_SUFFIX}; do\n") | ||
| file(APPEND "${_summary_script}" " if [ -x \"\$test\" ]; then\n") | ||
| file(APPEND "${_summary_script}" " TOTAL_FILES=\$((TOTAL_FILES + 1))\n") | ||
| file(APPEND "${_summary_script}" " TEST_BASENAME=\$(basename \"\$test\")\n") | ||
| file(APPEND "${_summary_script}" " # Get test names from --list-tests\n") | ||
| file(APPEND "${_summary_script}" " LIST_OUTPUT=\$(\"\$test\" --list-tests 2>&1)\n") | ||
| file(APPEND "${_summary_script}" " FILE_TOTAL=\$(echo \"\$LIST_OUTPUT\" | tail -1 | grep -o '^[0-9]*' || echo 0)\n") | ||
| file(APPEND "${_summary_script}" " TOTAL_TESTS=\$((TOTAL_TESTS + FILE_TOTAL))\n") | ||
| file(APPEND "${_summary_script}" " # Extract individual test names (lines starting with spaces after 'All available test cases:')\n") | ||
| file(APPEND "${_summary_script}" " TEST_NAMES=\$(echo \"\$LIST_OUTPUT\" | grep '^ ' | sed 's/^ //')\n") | ||
| file(APPEND "${_summary_script}" " # Parse the corresponding .test.out file for results\n") | ||
| file(APPEND "${_summary_script}" " OUT_FILE=\"Output/\${TEST_BASENAME}.test.out\"\n") | ||
| file(APPEND "${_summary_script}" " if [ -f \"\$OUT_FILE\" ]; then\n") | ||
| file(APPEND "${_summary_script}" " CASES_PASSED=0\n") | ||
| file(APPEND "${_summary_script}" " CASES_FAILED=0\n") | ||
| file(APPEND "${_summary_script}" " PARSED=0\n") | ||
| file(APPEND "${_summary_script}" " FILE_FAILED_NAMES=\"\"\n") | ||
| file(APPEND "${_summary_script}" " FILE_SKIPPED_NAMES=\"\"\n") | ||
| file(APPEND "${_summary_script}" " FILE_CRASHED_NAMES=\"\"\n") | ||
| file(APPEND "${_summary_script}" " # Parse Catch2 output - try multiple formats (prefer final summary over intermediate)\n") | ||
| file(APPEND "${_summary_script}" " # Format 1: 'All tests passed (N assertion in M test cases)' - final summary, most accurate\n") | ||
| file(APPEND "${_summary_script}" " ALL_PASSED_LINE=\$(grep 'All tests passed' \"\$OUT_FILE\" 2>/dev/null | tail -1 || echo \"\")\n") | ||
| file(APPEND "${_summary_script}" " if [ -n \"\$ALL_PASSED_LINE\" ]; then\n") | ||
| file(APPEND "${_summary_script}" " PARSED=1\n") | ||
| file(APPEND "${_summary_script}" " CASES_PASSED=\$(echo \"\$ALL_PASSED_LINE\" | grep -o 'in [0-9]* test case' | grep -o '[0-9]*' || echo 0)\n") | ||
| file(APPEND "${_summary_script}" " CASES_FAILED=0\n") | ||
| file(APPEND "${_summary_script}" " fi\n") | ||
| file(APPEND "${_summary_script}" " # Format 2: 'test cases: X | Y passed | Z failed' - use LAST occurrence (final summary)\n") | ||
| file(APPEND "${_summary_script}" " if [ \$PARSED -eq 0 ]; then\n") | ||
| file(APPEND "${_summary_script}" " SUMMARY_LINE=\$(grep '^test cases:' \"\$OUT_FILE\" 2>/dev/null | tail -1 || echo \"\")\n") | ||
| file(APPEND "${_summary_script}" " if [ -n \"\$SUMMARY_LINE\" ]; then\n") | ||
| file(APPEND "${_summary_script}" " PARSED=1\n") | ||
| file(APPEND "${_summary_script}" " CASES_PASSED=\$(echo \"\$SUMMARY_LINE\" | grep -o '[0-9]* passed' | grep -o '[0-9]*' || echo 0)\n") | ||
| file(APPEND "${_summary_script}" " CASES_FAILED=\$(echo \"\$SUMMARY_LINE\" | grep -o '[0-9]* failed' | grep -o '[0-9]*' || echo 0)\n") | ||
| file(APPEND "${_summary_script}" " CASES_TOTAL=\$(echo \"\$SUMMARY_LINE\" | sed 's/test cases: \\([0-9]*\\).*/\\1/')\n") | ||
| file(APPEND "${_summary_script}" " if [ \$CASES_PASSED -eq 0 ] && [ \$CASES_FAILED -gt 0 ]; then\n") | ||
| file(APPEND "${_summary_script}" " CASES_PASSED=\$((CASES_TOTAL - CASES_FAILED))\n") | ||
| file(APPEND "${_summary_script}" " fi\n") | ||
| file(APPEND "${_summary_script}" " fi\n") | ||
| file(APPEND "${_summary_script}" " fi\n") | ||
| file(APPEND "${_summary_script}" " # Update totals if we parsed something\n") | ||
| file(APPEND "${_summary_script}" " if [ \$PARSED -eq 1 ]; then\n") | ||
| file(APPEND "${_summary_script}" " # Detect runtime skips by parsing 'is skipped' messages\n") | ||
| file(APPEND "${_summary_script}" " RUNTIME_SKIPPED=\$(grep -i 'is skipped' \"\$OUT_FILE\" 2>/dev/null | wc -l)\n") | ||
| file(APPEND "${_summary_script}" " if [ \$RUNTIME_SKIPPED -gt 0 ] && [ \$CASES_PASSED -gt 0 ]; then\n") | ||
| file(APPEND "${_summary_script}" " if [ \$RUNTIME_SKIPPED -gt \$CASES_PASSED ]; then\n") | ||
| file(APPEND "${_summary_script}" " RUNTIME_SKIPPED=\$CASES_PASSED\n") | ||
| file(APPEND "${_summary_script}" " fi\n") | ||
| file(APPEND "${_summary_script}" " CASES_PASSED=\$((CASES_PASSED - RUNTIME_SKIPPED))\n") | ||
| file(APPEND "${_summary_script}" " SKIPPED_TESTS=\$((SKIPPED_TESTS + RUNTIME_SKIPPED))\n") | ||
| file(APPEND "${_summary_script}" " fi\n") | ||
| file(APPEND "${_summary_script}" " # Find failed and skipped test names by cross-referencing --list-tests with output\n") | ||
| file(APPEND "${_summary_script}" " # Catch2 console reporter shows 'TestName passed' for tests that passed\n") | ||
| file(APPEND "${_summary_script}" " FAILED_COUNT=\$CASES_FAILED\n") | ||
| file(APPEND "${_summary_script}" " SKIPPED_COUNT=\$RUNTIME_SKIPPED\n") | ||
| file(APPEND "${_summary_script}" " if [ -n \"\$TEST_NAMES\" ]; then\n") | ||
| file(APPEND "${_summary_script}" " while IFS= read -r tname; do\n") | ||
| file(APPEND "${_summary_script}" " [ -z \"\$tname\" ] && continue\n") | ||
| file(APPEND "${_summary_script}" " # Trim trailing whitespace from test name\n") | ||
| file(APPEND "${_summary_script}" " tname=\$(echo \"\$tname\" | sed 's/[[:space:]]*\$//')\n") | ||
| file(APPEND "${_summary_script}" " [ -z \"\$tname\" ] && continue\n") | ||
| file(APPEND "${_summary_script}" " # Check if this test passed\n") | ||
| file(APPEND "${_summary_script}" " if grep -q \"^\$tname passed\" \"\$OUT_FILE\" 2>/dev/null; then\n") | ||
| file(APPEND "${_summary_script}" " : # Test passed, nothing to track\n") | ||
| file(APPEND "${_summary_script}" " elif [ \$SKIPPED_COUNT -gt 0 ]; then\n") | ||
| file(APPEND "${_summary_script}" " # Test didn't pass - count as skipped first\n") | ||
| file(APPEND "${_summary_script}" " FILE_SKIPPED_NAMES=\"\${FILE_SKIPPED_NAMES}\${tname}|\"\n") | ||
| file(APPEND "${_summary_script}" " SKIPPED_COUNT=\$((SKIPPED_COUNT - 1))\n") | ||
| file(APPEND "${_summary_script}" " elif [ \$FAILED_COUNT -gt 0 ]; then\n") | ||
| file(APPEND "${_summary_script}" " # Test didn't pass and no more skips - count as failed\n") | ||
| file(APPEND "${_summary_script}" " FILE_FAILED_NAMES=\"\${FILE_FAILED_NAMES}\${tname}|\"\n") | ||
| file(APPEND "${_summary_script}" " FAILED_COUNT=\$((FAILED_COUNT - 1))\n") | ||
| file(APPEND "${_summary_script}" " fi\n") | ||
| file(APPEND "${_summary_script}" " done <<< \"\$TEST_NAMES\"\n") | ||
| file(APPEND "${_summary_script}" " fi\n") | ||
| file(APPEND "${_summary_script}" " PASSED_TESTS=\$((PASSED_TESTS + CASES_PASSED))\n") | ||
| file(APPEND "${_summary_script}" " FAILED_TESTS=\$((FAILED_TESTS + CASES_FAILED))\n") | ||
| file(APPEND "${_summary_script}" " # Categorize incomplete tests using exit code\n") | ||
| file(APPEND "${_summary_script}" " INCOMPLETE=\$((FILE_TOTAL - CASES_PASSED - CASES_FAILED - RUNTIME_SKIPPED))\n") | ||
| file(APPEND "${_summary_script}" " FILE_EXIT=\$(grep '^EXIT_CODE:' \"\$OUT_FILE\" 2>/dev/null | tail -1 | grep -o '[0-9]*' || echo 1)\n") | ||
| file(APPEND "${_summary_script}" " if [ \$INCOMPLETE -gt 0 ]; then\n") | ||
| file(APPEND "${_summary_script}" " if [ \"\$FILE_EXIT\" -eq 0 ]; then\n") | ||
| file(APPEND "${_summary_script}" " SKIPPED_TESTS=\$((SKIPPED_TESTS + INCOMPLETE))\n") | ||
| file(APPEND "${_summary_script}" " else\n") | ||
| file(APPEND "${_summary_script}" " CRASHED_TESTS=\$((CRASHED_TESTS + INCOMPLETE))\n") | ||
| file(APPEND "${_summary_script}" " # Track crashed test names - tests that didn't pass and weren't explicitly categorized\n") | ||
| file(APPEND "${_summary_script}" " if [ -n \"\$TEST_NAMES\" ]; then\n") | ||
| file(APPEND "${_summary_script}" " while IFS= read -r tname; do\n") | ||
| file(APPEND "${_summary_script}" " [ -z \"\$tname\" ] && continue\n") | ||
| file(APPEND "${_summary_script}" " tname=\$(echo \"\$tname\" | sed 's/[[:space:]]*\$//')\n") | ||
| file(APPEND "${_summary_script}" " [ -z \"\$tname\" ] && continue\n") | ||
| file(APPEND "${_summary_script}" " # Check if this test passed, was skipped, or was failed\n") | ||
| file(APPEND "${_summary_script}" " if ! grep -q \"^\$tname passed\" \"\$OUT_FILE\" 2>/dev/null; then\n") | ||
| file(APPEND "${_summary_script}" " # Not passed - check if already in skipped or failed\n") | ||
| file(APPEND "${_summary_script}" " if [[ \"\$FILE_SKIPPED_NAMES\" != *\"\$tname|\"* ]] && [[ \"\$FILE_FAILED_NAMES\" != *\"\$tname|\"* ]]; then\n") | ||
| file(APPEND "${_summary_script}" " FILE_CRASHED_NAMES=\"\${FILE_CRASHED_NAMES}\${tname}|\"\n") | ||
| file(APPEND "${_summary_script}" " fi\n") | ||
| file(APPEND "${_summary_script}" " fi\n") | ||
| file(APPEND "${_summary_script}" " done <<< \"\$TEST_NAMES\"\n") | ||
| file(APPEND "${_summary_script}" " fi\n") | ||
| file(APPEND "${_summary_script}" " fi\n") | ||
| file(APPEND "${_summary_script}" " fi\n") | ||
| file(APPEND "${_summary_script}" " # Accumulate test names for triage lists (names are | delimited)\n") | ||
| file(APPEND "${_summary_script}" " IFS='|' read -ra FAILED_ARR <<< \"\$FILE_FAILED_NAMES\"\n") | ||
| file(APPEND "${_summary_script}" " for name in \"\${FAILED_ARR[@]}\"; do\n") | ||
| file(APPEND "${_summary_script}" " [ -n \"\$name\" ] && FAILED_LIST=\"\${FAILED_LIST}\${name} [\${TEST_BASENAME}]\\n\"\n") | ||
| file(APPEND "${_summary_script}" " done\n") | ||
| file(APPEND "${_summary_script}" " IFS='|' read -ra SKIPPED_ARR <<< \"\$FILE_SKIPPED_NAMES\"\n") | ||
| file(APPEND "${_summary_script}" " for name in \"\${SKIPPED_ARR[@]}\"; do\n") | ||
| file(APPEND "${_summary_script}" " [ -n \"\$name\" ] && SKIPPED_LIST=\"\${SKIPPED_LIST}\${name} [\${TEST_BASENAME}]\\n\"\n") | ||
| file(APPEND "${_summary_script}" " done\n") | ||
| file(APPEND "${_summary_script}" " IFS='|' read -ra CRASHED_ARR <<< \"\$FILE_CRASHED_NAMES\"\n") | ||
| file(APPEND "${_summary_script}" " for name in \"\${CRASHED_ARR[@]}\"; do\n") | ||
| file(APPEND "${_summary_script}" " [ -n \"\$name\" ] && CRASHED_LIST=\"\${CRASHED_LIST}\${name} [\${TEST_BASENAME}]\\n\"\n") | ||
| file(APPEND "${_summary_script}" " done\n") | ||
| file(APPEND "${_summary_script}" " fi\n") | ||
| file(APPEND "${_summary_script}" " else\n") | ||
| file(APPEND "${_summary_script}" " # No output file - test crashed before producing output\n") | ||
| file(APPEND "${_summary_script}" " if [ \$FILE_TOTAL -gt 0 ]; then\n") | ||
| file(APPEND "${_summary_script}" " CRASHED_TESTS=\$((CRASHED_TESTS + FILE_TOTAL))\n") | ||
| file(APPEND "${_summary_script}" " # Add all test names to crashed list\n") | ||
| file(APPEND "${_summary_script}" " if [ -n \"\$TEST_NAMES\" ]; then\n") | ||
| file(APPEND "${_summary_script}" " while IFS= read -r tname; do\n") | ||
| file(APPEND "${_summary_script}" " [ -z \"\$tname\" ] && continue\n") | ||
| file(APPEND "${_summary_script}" " tname=\$(echo \"\$tname\" | sed 's/[[:space:]]*\$//')\n") | ||
| file(APPEND "${_summary_script}" " [ -n \"\$tname\" ] && CRASHED_LIST=\"\${CRASHED_LIST}\${tname} [\${TEST_BASENAME}]\\n\"\n") | ||
| file(APPEND "${_summary_script}" " done <<< \"\$TEST_NAMES\"\n") | ||
| file(APPEND "${_summary_script}" " fi\n") | ||
| file(APPEND "${_summary_script}" " fi\n") | ||
| file(APPEND "${_summary_script}" " fi\n") | ||
| file(APPEND "${_summary_script}" " fi\n") | ||
| file(APPEND "${_summary_script}" "done\n") | ||
| file(APPEND "${_summary_script}" "echo \" Test Suites: \$TOTAL_FILES\"\n") | ||
| file(APPEND "${_summary_script}" "echo \" Total Tests: \$TOTAL_TESTS\"\n") | ||
| file(APPEND "${_summary_script}" "echo \" Passed: \$PASSED_TESTS\"\n") | ||
| file(APPEND "${_summary_script}" "echo \" Failed: \$FAILED_TESTS\"\n") | ||
| file(APPEND "${_summary_script}" "if [ \$SKIPPED_TESTS -gt 0 ]; then\n") | ||
| file(APPEND "${_summary_script}" " echo \" Skipped: \$SKIPPED_TESTS\"\n") | ||
| file(APPEND "${_summary_script}" "fi\n") | ||
| file(APPEND "${_summary_script}" "if [ \$CRASHED_TESTS -gt 0 ]; then\n") | ||
| file(APPEND "${_summary_script}" " echo \" Crashed/Error: \$CRASHED_TESTS\"\n") | ||
| file(APPEND "${_summary_script}" "fi\n") | ||
| file(APPEND "${_summary_script}" "# Print triage lists\n") | ||
| file(APPEND "${_summary_script}" "if [ -n \"\$FAILED_LIST\" ]; then\n") | ||
| file(APPEND "${_summary_script}" " echo \"\"\n") | ||
| file(APPEND "${_summary_script}" " echo \"Failed Tests:\"\n") | ||
| file(APPEND "${_summary_script}" " echo -e \"\$FAILED_LIST\" | while IFS= read -r line; do\n") | ||
| file(APPEND "${_summary_script}" " [ -n \"\$line\" ] && echo \" - \$line\"\n") | ||
| file(APPEND "${_summary_script}" " done\n") | ||
| file(APPEND "${_summary_script}" "fi\n") | ||
| file(APPEND "${_summary_script}" "if [ -n \"\$SKIPPED_LIST\" ]; then\n") | ||
| file(APPEND "${_summary_script}" " echo \"\"\n") | ||
| file(APPEND "${_summary_script}" " echo \"Skipped Tests:\"\n") | ||
| file(APPEND "${_summary_script}" " echo -e \"\$SKIPPED_LIST\" | while IFS= read -r line; do\n") | ||
| file(APPEND "${_summary_script}" " [ -n \"\$line\" ] && echo \" - \$line\"\n") | ||
| file(APPEND "${_summary_script}" " done\n") | ||
| file(APPEND "${_summary_script}" "fi\n") | ||
| file(APPEND "${_summary_script}" "if [ -n \"\$CRASHED_LIST\" ]; then\n") | ||
| file(APPEND "${_summary_script}" " echo \"\"\n") | ||
| file(APPEND "${_summary_script}" " echo \"Crashed/Error Tests:\"\n") | ||
| file(APPEND "${_summary_script}" " echo -e \"\$CRASHED_LIST\" | while IFS= read -r line; do\n") | ||
| file(APPEND "${_summary_script}" " [ -n \"\$line\" ] && echo \" - \$line\"\n") | ||
| file(APPEND "${_summary_script}" " done\n") | ||
| file(APPEND "${_summary_script}" "fi\n") | ||
| file(APPEND "${_summary_script}" "echo \"========================================\"\n") | ||
| execute_process(COMMAND chmod +x "${_summary_script}") |
Member
There was a problem hiding this comment.
I'm not super familiar with cmake, but this looks odd to me, is this really common practice for providing a test script? Embedding it like this in the cmake file looks like it might hurt maintainability.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
2 participants
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Introduce support for HIP Catch tests in
External/HIPtests.HIP Catch tests-DENABLE_HIP_CATCH_TESTS=ONin cmakeSee External/HIP/CATCH_TESTS_README.md for details