From 91b8969ab4c995521eefcfe0a2f4f48613e06e35 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Fri, 8 May 2026 11:59:20 +0000 Subject: [PATCH 1/6] [INFRA] Share Maven precompile artifact across maven_test matrix Follow-up to SPARK-56768. Adds a `precompile-maven` job to `maven_test.yml` that runs `mvn clean install -DskipTests` once and publishes the resulting `target/` trees plus `~/.m2/repository/org/apache/spark/` as a GitHub Actions artifact. Each of the 12 matrix entries now consumes that artifact instead of running its own `mvn clean install` from scratch. The Maven version of the optimization differs from the SBT one in two places: 1. We tar two pieces and upload as a single multi-file artifact: `compile-target.tar.gz` (workspace target/ trees) and `compile-m2-spark.tar.gz` (the Spark portion of the local Maven repository, needed for cross-module dependency resolution at `mvn -pl X test` time). 2. The artifact name is JDK-tagged `spark-maven-compile--java-` because the build_maven_*.yml callers use different JDKs (17, 21, 25) and each produces non-interchangeable bytecode. Same optional/fallback design as SPARK-56768: - `precompile-maven` is `continue-on-error: true`; a failure does not fail the workflow run. - The matrix uses `if: (!cancelled())` so it runs even on precompile failure or cancellation. - The "Download precompiled artifact" step is gated on `needs.precompile-maven.result == 'success'` and has `continue-on-error: true`. - The "Extract precompiled artifact" step is gated on the download succeeding. - Inside the "Run tests" bash, the local `mvn clean install` is run only when `steps.extract-precompiled.outcome != 'success'`. Otherwise the artifact's classes/jars are used directly. Logs `Reusing precompiled artifact, skipping local Maven clean install.` for visibility. The hive-thriftserver special case (line ~228, "To avoid a compilation loop") still does its own `clean install` and is not touched by this PR; it does ~1 of 12 entries' worth of redundant work, which is acceptable. Estimated saving: roughly 11 of the 12 matrix entries skip ~25-40m of Maven clean install each; netting ~300m+ of CI compute saved per scheduled run, per JDK. Generated-by: Claude Code (Opus 4.7) --- .github/workflows/maven_test.yml | 106 ++++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index 357d869d1b88c..1d1bf3d185250 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -52,9 +52,90 @@ on: type: string default: '{}' jobs: + # Precompile Spark with Maven once and publish target/ + ~/.m2/.../spark as + # an artifact for the matrix entries below to consume. Optional: any failure + # here degrades the matrix to its original local `clean install` path. + precompile-maven: + name: "Precompile Spark with Maven" + runs-on: ${{ inputs.os }} + # If this job fails or is cancelled, the matrix entries fall back to + # running `mvn clean install` locally as before. + continue-on-error: true + env: + HADOOP_PROFILE: ${{ inputs.hadoop }} + HIVE_PROFILE: hive2.3 + SPARK_LOCAL_IP: localhost + GITHUB_PREV_SHA: ${{ github.event.before }} + steps: + - name: Checkout Spark repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + repository: apache/spark + ref: ${{ inputs.branch }} + - name: Sync the current branch with the latest in Apache Spark + if: github.repository != 'apache/spark' + run: | + echo "APACHE_SPARK_REF=$(git rev-parse HEAD)" >> $GITHUB_ENV + git fetch https://github.com/$GITHUB_REPOSITORY.git ${GITHUB_REF#refs/heads/} + git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD + git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty + - name: Cache SBT and Maven + if: ${{ runner.os != 'macOS' }} + uses: actions/cache@v5 + with: + path: | + build/apache-maven-* + build/*.jar + ~/.sbt + key: build-${{ hashFiles('**/pom.xml', 'project/build.properties', 'build/mvn', 'build/sbt', 'build/sbt-launch-lib.bash', 'build/spark-build-info') }} + restore-keys: | + build- + - name: Cache Maven local repository + if: ${{ runner.os != 'macOS' }} + uses: actions/cache@v5 + with: + path: ~/.m2/repository + key: java${{ inputs.java }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + java${{ inputs.java }}-maven- + - name: Install Java ${{ inputs.java }} + uses: actions/setup-java@v5 + with: + distribution: zulu + java-version: ${{ inputs.java }} + - name: Build Spark with Maven + shell: | + bash -c "if script -qec true 2>/dev/null; then script -qec bash\ {0}; else script -qe /dev/null bash {0}; fi" + run: | + set -e + export MAVEN_OPTS="-Xss64m -Xmx4g -Xms4g -XX:ReservedCodeCacheSize=128m -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" + export MAVEN_CLI_OPTS="--no-transfer-progress" + export JAVA_VERSION=${{ inputs.java }} + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} clean install + - name: Package compile output + run: | + find . -type d -name target -not -path './build/*' -not -path './.git/*' -print0 \ + | tar --null -czf compile-target.tar.gz -T - + if [ -d "$HOME/.m2/repository/org/apache/spark" ]; then + tar -C "$HOME/.m2/repository/org/apache" -czf compile-m2-spark.tar.gz spark + fi + ls -lh compile-target.tar.gz compile-m2-spark.tar.gz + - name: Upload compile artifact + uses: actions/upload-artifact@v6 + with: + name: spark-maven-compile-${{ inputs.branch }}-java${{ inputs.java }}-${{ github.run_id }} + path: | + compile-target.tar.gz + compile-m2-spark.tar.gz + retention-days: 1 + if-no-files-found: error + # Build: build Spark and run the tests for specified modules using maven build: name: "Build modules: ${{ matrix.modules }} ${{ matrix.comment }}" + needs: precompile-maven + if: (!cancelled()) runs-on: ${{ inputs.os }} # TODO(SPARK-54466): https://github.com/actions/runner-images/issues/13341 # timeout-minutes: 150 @@ -184,6 +265,25 @@ jobs: run: | python3.12 -m pip install 'numpy>=1.23.2' pyarrow 'pandas==2.3.3' pyyaml scipy unittest-xml-reporting 'grpcio==1.76.0' 'grpcio-status==1.76.0' 'protobuf==6.33.5' 'zstandard==0.25.0' python3.12 -m pip list + - name: Download precompiled artifact + id: download-precompiled + if: needs.precompile-maven.result == 'success' + continue-on-error: true + uses: actions/download-artifact@v6 + with: + name: spark-maven-compile-${{ inputs.branch }}-java${{ matrix.java }}-${{ github.run_id }} + - name: Extract precompiled artifact + id: extract-precompiled + if: steps.download-precompiled.outcome == 'success' + continue-on-error: true + run: | + tar -xzf compile-target.tar.gz + rm compile-target.tar.gz + if [ -f compile-m2-spark.tar.gz ]; then + mkdir -p "$HOME/.m2/repository/org/apache" + tar -C "$HOME/.m2/repository/org/apache" -xzf compile-m2-spark.tar.gz + rm compile-m2-spark.tar.gz + fi # Run the tests using script command. # BSD's script command doesn't support -c option, and the usage is different from Linux's one. # The kind of script command is tested by `script -qec true`. @@ -203,7 +303,11 @@ jobs: export ENABLE_KINESIS_TESTS=0 # Replace with the real module name, for example, connector#kafka-0-10 -> connector/kafka-0-10 export TEST_MODULES=`echo "$MODULES_TO_TEST" | sed -e "s%#%/%g"` - ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} clean install + if [ "${{ steps.extract-precompiled.outcome }}" = "success" ]; then + echo "Reusing precompiled artifact, skipping local Maven clean install." + else + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} clean install + fi if [ "$MODULES_TO_TEST" != "connect" ]; then echo "Clean up the assembly module before maven testing" From 1732951a3cd79d88e9f0e72734862dd0f4658c6e Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Fri, 8 May 2026 12:12:27 +0000 Subject: [PATCH 2/6] [INFRA] Exclude assembly from Maven precompile artifact, rebuild for connect Eleven of the 12 matrix entries wipe assembly/target/ immediately after extraction via the existing `mvn clean -pl assembly` step (SPARK-51628, which exists to keep the SPARK-51600 fix path covered by the daily Maven test). Including assembly in the artifact wastes upload + download bandwidth for those 11 entries. This commit: 1. Excludes `assembly/` from the find pattern in the precompile-maven "Package compile output" step. Uses `-prune` so any nested target/ dirs under assembly are also excluded. 2. Adds an explicit `mvn install -pl assembly` step in the matrix entry's bash, gated on `MODULES_TO_TEST = "connect"` and the artifact reuse path. The connect entry is the only one that needs the assembly built (SPARK-51628 leaves it out of the cleanup for that reason); now we build it on demand instead of carrying it around for entries that throw it away. The SPARK-51628 cleanup step (`mvn clean -pl assembly` for non-connect) still runs and is now a near-no-op for the reuse path; it remains a correctness guard for the fallback path that does run `clean install`. Generated-by: Claude Code (Opus 4.7) --- .github/workflows/maven_test.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index 1d1bf3d185250..e17e2271ec670 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -115,7 +115,11 @@ jobs: ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} clean install - name: Package compile output run: | - find . -type d -name target -not -path './build/*' -not -path './.git/*' -print0 \ + # Exclude assembly/ from the artifact: 11 of 12 matrix entries wipe it + # right after extraction (SPARK-51628 regression-test-for-SPARK-51600), + # and the connect entry rebuilds it via `mvn install -pl assembly`. + find . \( -path './build' -o -path './.git' -o -path './assembly' \) -prune \ + -o -type d -name target -print0 \ | tar --null -czf compile-target.tar.gz -T - if [ -d "$HOME/.m2/repository/org/apache/spark" ]; then tar -C "$HOME/.m2/repository/org/apache" -czf compile-m2-spark.tar.gz spark @@ -305,6 +309,12 @@ jobs: export TEST_MODULES=`echo "$MODULES_TO_TEST" | sed -e "s%#%/%g"` if [ "${{ steps.extract-precompiled.outcome }}" = "success" ]; then echo "Reusing precompiled artifact, skipping local Maven clean install." + # The assembly module is excluded from the artifact (see precompile-maven). + # Connect tests strongly depend on a built assembly module; rebuild it here. + if [ "$MODULES_TO_TEST" = "connect" ]; then + echo "Building assembly module for connect tests." + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} -pl assembly install + fi else ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} clean install fi From c57f23c52928a9010e6319a031d348bbc8eff648 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Fri, 8 May 2026 12:14:10 +0000 Subject: [PATCH 3/6] [INFRA] Add SPARK-54466 TODO to precompile-maven cache steps Mirror the comment used on the existing matrix-job cache steps so a future maintainer knows the macOS gate on these new cache steps is a workaround for the upstream GHA hashFiles failure tracked in SPARK-54466 / actions/runner-images#13341, and can be removed once that issue is resolved. Generated-by: Claude Code (Opus 4.7) --- .github/workflows/maven_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index e17e2271ec670..cd647dd181bac 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -81,6 +81,7 @@ jobs: git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' merge --no-commit --progress --squash FETCH_HEAD git -c user.name='Apache Spark Test Account' -c user.email='sparktestacc@gmail.com' commit -m "Merged commit" --allow-empty - name: Cache SBT and Maven + # TODO(SPARK-54466): https://github.com/actions/runner-images/issues/13341 if: ${{ runner.os != 'macOS' }} uses: actions/cache@v5 with: @@ -92,6 +93,7 @@ jobs: restore-keys: | build- - name: Cache Maven local repository + # TODO(SPARK-54466): https://github.com/actions/runner-images/issues/13341 if: ${{ runner.os != 'macOS' }} uses: actions/cache@v5 with: From 0031878d4cf8da3201ed5adb72b1637eb44bfeda Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Fri, 8 May 2026 12:26:07 +0000 Subject: [PATCH 4/6] [INFRA] Move SPARK-51628 assembly cleanup into the fallback branch The `mvn clean -pl assembly` step exists to wipe assembly/target/ so tests exercise the SPARK-51600 prepend fallback. On the precompile reuse path the assembly module is already excluded from the artifact, so the cleanup is a no-op (~5-10s of wasted Maven invocation per non-connect entry, ~50-100s per scheduled run). Move the cleanup into the fallback branch, where it's still needed. The reuse path's regression coverage is preserved by the artifact having no assembly to begin with. Generated-by: Claude Code (Opus 4.7) --- .github/workflows/maven_test.yml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/maven_test.yml b/.github/workflows/maven_test.yml index cd647dd181bac..0799d871e6d66 100644 --- a/.github/workflows/maven_test.yml +++ b/.github/workflows/maven_test.yml @@ -311,21 +311,26 @@ jobs: export TEST_MODULES=`echo "$MODULES_TO_TEST" | sed -e "s%#%/%g"` if [ "${{ steps.extract-precompiled.outcome }}" = "success" ]; then echo "Reusing precompiled artifact, skipping local Maven clean install." - # The assembly module is excluded from the artifact (see precompile-maven). - # Connect tests strongly depend on a built assembly module; rebuild it here. + # SPARK-51628 regression coverage is naturally preserved on the reuse + # path: the precompile artifact excludes assembly/, so non-connect + # tests already run with the assembly module's jars dir missing. + # Connect tests strongly depend on a built assembly module; rebuild + # it here. if [ "$MODULES_TO_TEST" = "connect" ]; then echo "Building assembly module for connect tests." ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} -pl assembly install fi else ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} clean install + # SPARK-51628: wipe the assembly module so tests exercise the + # SPARK-51600 prepend fallback path. Connect tests strongly depend + # on a built assembly module, so they are excluded. + if [ "$MODULES_TO_TEST" != "connect" ]; then + echo "Clean up the assembly module before maven testing" + ./build/mvn $MAVEN_CLI_OPTS clean -pl assembly + fi fi - - if [ "$MODULES_TO_TEST" != "connect" ]; then - echo "Clean up the assembly module before maven testing" - ./build/mvn $MAVEN_CLI_OPTS clean -pl assembly - fi - + if [[ "$INCLUDED_TAGS" != "" ]]; then ./build/mvn $MAVEN_CLI_OPTS -pl "$TEST_MODULES" -Pyarn -Pkubernetes -Pvolcano -Phive -Phive-thriftserver -Phadoop-cloud -Pjvm-profiler -Pspark-ganglia-lgpl -Pkinesis-asl -Djava.version=${JAVA_VERSION/-ea} -Dtest.include.tags="$INCLUDED_TAGS" test -fae elif [[ "$MODULES_TO_TEST" == "connect" && "$INPUT_BRANCH" == "branch-4.0" ]]; then From c43e36c629fa5118faebd52c40b47e3038df8dc3 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Fri, 8 May 2026 12:26:41 +0000 Subject: [PATCH 5/6] [INFRA][TEMP] Trigger build_maven.yml on push to validate this PR REVERT BEFORE MERGE. Adds `push:` to the trigger list and removes the `if: github.repository == 'apache/spark'` job-level gate so each push to this branch on the fork fires build_maven.yml. This exercises maven_test.yml end-to-end with the precompile-maven changes from this PR. Generated-by: Claude Code (Opus 4.7) --- .github/workflows/build_maven.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_maven.yml b/.github/workflows/build_maven.yml index c177fcf014419..8ee386fa8b143 100644 --- a/.github/workflows/build_maven.yml +++ b/.github/workflows/build_maven.yml @@ -20,6 +20,11 @@ name: "Build / Maven (master, Scala 2.13, Hadoop 3, JDK 17)" on: + # TEMP for PR #55766: also run on every push to validate maven_test.yml + # changes from this PR. REVERT BEFORE MERGE. + push: + branches: + - '**' schedule: - cron: '0 13 */2 * *' workflow_dispatch: @@ -30,4 +35,5 @@ jobs: packages: write name: Run uses: ./.github/workflows/maven_test.yml - if: github.repository == 'apache/spark' + # TEMP for PR #55766: removed `if: github.repository == 'apache/spark'` so + # this workflow fires on the fork. REVERT BEFORE MERGE. From 7d99f4e8fc169886e2f0c2d57594e13d5d186dbd Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Thu, 21 May 2026 11:03:36 +0000 Subject: [PATCH 6/6] Revert "[INFRA][TEMP] Trigger build_maven.yml on push to validate this PR" This reverts commit c43e36c629fa5118faebd52c40b47e3038df8dc3. --- .github/workflows/build_maven.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/build_maven.yml b/.github/workflows/build_maven.yml index 8ee386fa8b143..c177fcf014419 100644 --- a/.github/workflows/build_maven.yml +++ b/.github/workflows/build_maven.yml @@ -20,11 +20,6 @@ name: "Build / Maven (master, Scala 2.13, Hadoop 3, JDK 17)" on: - # TEMP for PR #55766: also run on every push to validate maven_test.yml - # changes from this PR. REVERT BEFORE MERGE. - push: - branches: - - '**' schedule: - cron: '0 13 */2 * *' workflow_dispatch: @@ -35,5 +30,4 @@ jobs: packages: write name: Run uses: ./.github/workflows/maven_test.yml - # TEMP for PR #55766: removed `if: github.repository == 'apache/spark'` so - # this workflow fires on the fork. REVERT BEFORE MERGE. + if: github.repository == 'apache/spark'