diff --git a/.github/actions/get-hugegraph-commit/action.yml b/.github/actions/get-hugegraph-commit/action.yml index b07b45c9e..60152b0d1 100644 --- a/.github/actions/get-hugegraph-commit/action.yml +++ b/.github/actions/get-hugegraph-commit/action.yml @@ -47,7 +47,7 @@ runs: core.exportVariable('COMMIT_REF', result.fetchRef); core.setOutput('commit_id', result.sha); core.setOutput('fetch_ref', result.fetchRef); - core.info(`Server baseline: ${process.env.SERVER_REPOSITORY} ${result.fetchRef} (${result.sha})`); + core.info(`Server baseline: ${process.env.SERVER_REPOSITORY} ${/^[0-9a-f]{40}$/i.test(result.fetchRef) ? result.fetchRef.slice(0, 6) : result.fetchRef} (${result.sha.slice(0, 6)})`); } catch (error) { core.setFailed(`Failed to get HugeGraph commit: ${error.message}`); } diff --git a/.github/actions/get-hugegraph-commit/resolve.cjs b/.github/actions/get-hugegraph-commit/resolve.cjs index 050637950..ae0ea897b 100644 --- a/.github/actions/get-hugegraph-commit/resolve.cjs +++ b/.github/actions/get-hugegraph-commit/resolve.cjs @@ -45,8 +45,16 @@ module.exports = async function resolve(github, { repository, ref = '', pr = '', sha = commit.sha; } if (!/^[0-9a-f]{40}$/i.test(sha)) throw new Error('Server API returned an invalid commit SHA'); - if (expectedCommit && sha.toLowerCase() !== expectedCommit.toLowerCase()) { - throw new Error(`Server ref moved: expected ${expectedCommit}, resolved ${sha}`); + if (expectedCommit) { + if (sha.toLowerCase() !== expectedCommit.toLowerCase()) { + throw new Error(`Server ref moved: expected ${expectedCommit.slice(0, 6)}, resolved ${sha.slice(0, 6)}`); + } + } + if (/^[0-9a-f]{40}$/i.test(ref)) { + if (sha.toLowerCase() !== ref.toLowerCase()) { + throw new Error('Server API returned a commit different from the selected SHA'); + } + fetchRef = sha; } return { sha, fetchRef }; }; diff --git a/.github/actions/get-hugegraph-commit/resolve.test.cjs b/.github/actions/get-hugegraph-commit/resolve.test.cjs index 788be95a5..62ceb0b9c 100644 --- a/.github/actions/get-hugegraph-commit/resolve.test.cjs +++ b/.github/actions/get-hugegraph-commit/resolve.test.cjs @@ -57,7 +57,7 @@ test('resolves PR head from the selected repository', async () => { test('rejects invalid or ambiguous selectors before calling GitHub', async () => { for (const input of [{ repository: '../repo' }, { ref: 'master', pr: '42' }, - { pr: '1.5' }, { pr: '0' }, { expectedCommit: 'short' }]) { + { pr: '1.5' }, { pr: '0' }, { expectedCommit: 'short' }, { expectedCommit: sha.slice(0, 6) }]) { const { github, calls } = client(); await assert.rejects(resolve(github, { repository: 'apache/hugegraph', ...input })); assert.equal(calls.length, 0); @@ -71,3 +71,12 @@ test('fails closed for a moved branch or API error', async () => { github.rest.repos.getCommit = async () => { throw new Error('Not Found'); }; await assert.rejects(resolve(github, { repository: 'example/server', ref: 'missing' }), /Not Found/); }); + +test('verifies a full commit selector and fetch identity', async () => { + const { github, calls } = client(); + assert.deepEqual(await resolve(github, { repository: 'apache/hugegraph', + ref: sha, expectedCommit: sha }), { sha, fetchRef: sha }); + assert.deepEqual(calls, [['commit', { owner: 'apache', repo: 'hugegraph', ref: sha }]]); + await assert.rejects(resolve(github, { repository: 'apache/hugegraph', + ref: 'b'.repeat(40) }), /different from the selected SHA/); +}); diff --git a/.github/actions/setup-candidate-sdk/action.yml b/.github/actions/setup-candidate-sdk/action.yml index b6652a8c1..02a52e556 100644 --- a/.github/actions/setup-candidate-sdk/action.yml +++ b/.github/actions/setup-candidate-sdk/action.yml @@ -6,11 +6,11 @@ inputs: description: 'GitHub owner/repository for the candidate SDK' default: 'apache/hugegraph' ref: - description: 'Fetchable candidate source ref' - default: 'd9abcd4317fb36128e7e4d209139ec7f3a28cdfc' + description: 'Immutable candidate source commit' + default: 'e62c961e00221569d4f955abbadf60faee45b283' expected-commit: - description: 'Locked full candidate source commit' - default: 'd9abcd4317fb36128e7e4d209139ec7f3a28cdfc' + description: 'Locked candidate source commit' + default: 'e62c961e00221569d4f955abbadf60faee45b283' outputs: repository: @@ -55,9 +55,18 @@ runs: run: | candidate_repo=$(mktemp -d "$RUNNER_TEMP/hugegraph-candidate-m2.XXXXXX") server_build=$(mktemp -d "$RUNNER_TEMP/hugegraph-candidate-build.XXXXXX") + source_dir="$server_build/source" + if [[ -d "$HOME/.m2/repository" ]]; then + rsync -a --exclude='/org/apache/hugegraph/' --exclude='/candidate-sdk-manifest.json' \ + "$HOME/.m2/repository/" "$candidate_repo/" + fi bash "$GITHUB_WORKSPACE/hugegraph-client/assembly/travis/install-candidate-sdk.sh" \ - "$SERVER_COMMIT" "$server_build/hugegraph" "$candidate_repo" - archives=("$server_build"/hugegraph/hugegraph-server/apache-hugegraph-*.tar.gz) + "$SERVER_COMMIT" "$source_dir" "$candidate_repo" + # Retain downloaded third-party inputs for the existing Maven cache. + mkdir -p "$HOME/.m2/repository" + rsync -a --exclude='/org/apache/hugegraph/' --exclude='/candidate-sdk-manifest.json' \ + "$candidate_repo/" "$HOME/.m2/repository/" + archives=("$source_dir"/hugegraph-server/apache-hugegraph-*.tar.gz) if [[ ${#archives[@]} -ne 1 || ! -f "${archives[0]}" ]]; then printf 'Expected one candidate server archive, found: %s\n' "${archives[*]}" >&2 exit 1 diff --git a/.github/actions/setup-hugegraph-server/action.yml b/.github/actions/setup-hugegraph-server/action.yml index d80ec1d96..979a5f19d 100644 --- a/.github/actions/setup-hugegraph-server/action.yml +++ b/.github/actions/setup-hugegraph-server/action.yml @@ -1,5 +1,5 @@ name: Setup HugeGraph Server -description: Build an immutable fixture once, or start an independent test service +description: Prepare an immutable release or candidate fixture, or start an independent test service inputs: mode: description: 'standalone (legacy), build (upload), or start (download)' @@ -8,17 +8,20 @@ inputs: description: Workflow-local immutable fixture artifact name default: '' repository: - description: GitHub owner/repository containing the Server source + description: Server source repository (Java 11 release requires apache/hugegraph) default: apache/hugegraph fetch-ref: description: Fetchable source ref (must resolve to commit-id) default: '' commit-id: - description: Exact Server commit SHA + description: Exact Server commit SHA (Java 11 release requires the locked 1.7.0 commit) required: true java-version: description: JDK major used for the build and runtime default: '11' + release-version: + description: Fixed official Server release used by Java 11 compatibility fixtures + default: '1.7.0' java-home: description: Optional fixture JDK scoped to this subprocess; keeps Toolchain JDK unchanged default: '' @@ -37,12 +40,12 @@ runs: with: name: ${{ inputs.artifact-name }} path: ${{ runner.temp }}/hugegraph-fixture - - name: Restore exact Server build cache + - name: Restore exact Server fixture cache if: inputs.mode == 'build' uses: actions/cache@v4 with: path: ~/hugegraph-fixture-cache - key: server-fixture-v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.repository }}-${{ inputs.commit-id }}-jdk${{ inputs.java-version }}-${{ hashFiles('.github/actions/setup-hugegraph-server/action.yml', '.github/actions/setup-hugegraph-server/fixture.sh', '.github/actions/setup-hugegraph-server/manifest.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh') }} + key: server-fixture-v2-${{ runner.os }}-${{ runner.arch }}-${{ inputs.repository }}-${{ inputs.commit-id }}-release${{ inputs.release-version }}-jdk${{ inputs.java-version }}-${{ hashFiles('.github/actions/setup-hugegraph-server/action.yml', '.github/actions/setup-hugegraph-server/fixture.sh', '.github/actions/setup-hugegraph-server/manifest.py', '.github/actions/setup-hugegraph-server/release.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', 'hugegraph-client/assembly/travis/install-candidate-sdk.sh', 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh') }} - name: Prepare fixture or start service shell: bash env: @@ -50,9 +53,10 @@ runs: FIXTURE_REPOSITORY: ${{ inputs.repository }} FIXTURE_COMMIT: ${{ inputs.commit-id }} FIXTURE_REF: ${{ inputs.fetch-ref }} - FIXTURE_BUILD_INPUTS: ${{ hashFiles('.github/actions/setup-hugegraph-server/action.yml', '.github/actions/setup-hugegraph-server/fixture.sh', '.github/actions/setup-hugegraph-server/manifest.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh') }} + FIXTURE_BUILD_INPUTS: ${{ hashFiles('.github/actions/setup-hugegraph-server/action.yml', '.github/actions/setup-hugegraph-server/fixture.sh', '.github/actions/setup-hugegraph-server/manifest.py', '.github/actions/setup-hugegraph-server/release.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', 'hugegraph-client/assembly/travis/install-candidate-sdk.sh', 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh') }} FIXTURE_PLATFORM: ${{ runner.os }}-${{ runner.arch }} FIXTURE_JAVA: ${{ inputs.java-version }} + FIXTURE_RELEASE_VERSION: ${{ inputs.release-version }} FIXTURE_JAVA_HOME: ${{ inputs.java-home }} ISOLATE_FIXTURE_REPO: ${{ inputs.isolate-maven-repo }} FIXTURE_DIR: ${{ runner.temp }}/hugegraph-fixture @@ -85,6 +89,10 @@ runs: export TRAVIS_DIR server_build=$(mktemp -d "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/hugegraph-build.XXXXXX") cd "$server_build" + if [[ "$FIXTURE_MODE" == standalone ]]; then + FIXTURE_MODE=build bash "$FIXTURE_ACTION/fixture.sh" + export FIXTURE_MODE=start + fi if [[ -f "$TRAVIS_DIR/install-hugegraph-from-source.sh" ]]; then bash "$TRAVIS_DIR/install-hugegraph-from-source.sh" "$FIXTURE_COMMIT" else diff --git a/.github/actions/setup-hugegraph-server/fixture.sh b/.github/actions/setup-hugegraph-server/fixture.sh index 9fc5f1850..375045811 100644 --- a/.github/actions/setup-hugegraph-server/fixture.sh +++ b/.github/actions/setup-hugegraph-server/fixture.sh @@ -22,15 +22,29 @@ FIXTURE_REPOSITORY=${FIXTURE_REPOSITORY:-${SERVER_REPOSITORY:-apache/hugegraph}} FIXTURE_COMMIT=${FIXTURE_COMMIT:-${1:-}} FIXTURE_REF=${FIXTURE_REF:-$FIXTURE_COMMIT} FIXTURE_JAVA=${FIXTURE_JAVA:-11} +FIXTURE_RELEASE_VERSION=${FIXTURE_RELEASE_VERSION:-1.7.0} FIXTURE_DIR=${FIXTURE_DIR:-${RUNNER_TEMP:-${TMPDIR:-/tmp}}/hugegraph-fixture} FIXTURE_HELPER=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/manifest.py FIXTURE_BUILD_INPUTS=${FIXTURE_BUILD_INPUTS:-$(python3 "$FIXTURE_HELPER" inputs)} FIXTURE_PLATFORM=${FIXTURE_PLATFORM:-$(uname -sm)} -FIXTURE_CONFIG="package -DskipTests -Dmaven.javadoc.skip=true -ntp;schema=1;inputs=$FIXTURE_BUILD_INPUTS;platform=$FIXTURE_PLATFORM" -export FIXTURE_REPOSITORY FIXTURE_COMMIT FIXTURE_JAVA FIXTURE_CONFIG FIXTURE_DIR +if [[ "$FIXTURE_JAVA" == 17 ]]; then + FIXTURE_BUILD_COMMAND=candidate-sdk-install +else + FIXTURE_BUILD_COMMAND="asf-release-$FIXTURE_RELEASE_VERSION" +fi +FIXTURE_CONFIG="$FIXTURE_BUILD_COMMAND;schema=2;inputs=$FIXTURE_BUILD_INPUTS;platform=$FIXTURE_PLATFORM" +export FIXTURE_REPOSITORY FIXTURE_COMMIT FIXTURE_JAVA FIXTURE_CONFIG FIXTURE_DIR FIXTURE_RELEASE_VERSION [[ "$FIXTURE_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] [[ "$FIXTURE_COMMIT" =~ ^[0-9a-f]{40}$ ]] -[[ "$FIXTURE_JAVA" =~ ^[0-9]+$ ]] +[[ "$FIXTURE_JAVA" == 11 || "$FIXTURE_JAVA" == 17 ]] || { + echo 'Fixture requires JDK 11 for the official release or JDK 17 for the candidate SDK' >&2 + exit 1 +} + +if [[ "$FIXTURE_JAVA" == 11 ]]; then + # Reject caller/source mismatch before restoring a cache or downloading bytes. + python3 "${FIXTURE_HELPER%/*}/release.py" validate +fi fixture_validate_java() { actual_java=$(java -version 2>&1 | sed -n 's/.*version "\([0-9][0-9]*\).*".*/\1/p' | head -1) @@ -39,7 +53,7 @@ fixture_validate_java() { fixture_verify() { python3 "$FIXTURE_HELPER" verify; } fixture_build() { - local cache_key cache_dir source_dir archive + local cache_key cache_dir source_dir archive candidate_repo='' cache_key=$(printf '%s\n' "$FIXTURE_REPOSITORY" "$FIXTURE_COMMIT" "$FIXTURE_JAVA" "$FIXTURE_CONFIG" | shasum -a 256 | cut -d ' ' -f 1) cache_dir="${HOME}/hugegraph-fixture-cache/$cache_key" mkdir -p "$FIXTURE_DIR" @@ -50,19 +64,27 @@ fixture_build() { fi rm -f "$FIXTURE_DIR/server.tar.gz" "$FIXTURE_DIR/manifest.json" fixture_validate_java - source_dir="$(mktemp -d "${TMPDIR:-/tmp}/hg-fixture-source.XXXXXX")/source" - SERVER_REPOSITORY=$FIXTURE_REPOSITORY SERVER_FETCH_REF=$FIXTURE_REF \ - bash "${FIXTURE_HELPER%/*}/../../../hugegraph-client/assembly/travis/checkout-server.sh" "$FIXTURE_COMMIT" "$source_dir" - (cd "$source_dir" && mvn package -DskipTests -Dmaven.javadoc.skip=true -ntp) - archive=$(find "$source_dir/hugegraph-server" -maxdepth 1 -name 'apache-hugegraph-*.tar.gz' -print) - [[ -n "$archive" && -f "$archive" ]] - cp "$archive" "$FIXTURE_DIR/server.tar.gz" - export FIXTURE_ARCHIVE_NAME=${archive##*/} + if [[ "$FIXTURE_JAVA" == 17 ]]; then + source_dir="$(mktemp -d "${TMPDIR:-/tmp}/hg-fixture-source.XXXXXX")/source" + # Reuse the verified SDK build's same-source distribution and isolated repository. + candidate_repo=$(mktemp -d "${TMPDIR:-/tmp}/hg-fixture-sdk-m2.XXXXXX") + SERVER_REPOSITORY=$FIXTURE_REPOSITORY SERVER_FETCH_REF=$FIXTURE_REF \ + bash "${FIXTURE_HELPER%/*}/../../../hugegraph-client/assembly/travis/install-candidate-sdk.sh" \ + "$FIXTURE_COMMIT" "$source_dir" "$candidate_repo" + archive=$(find "$source_dir/hugegraph-server" -maxdepth 1 -name 'apache-hugegraph-*.tar.gz' -print) + [[ -n "$archive" && -f "$archive" ]] + cp "$archive" "$FIXTURE_DIR/server.tar.gz" + export FIXTURE_ARCHIVE_NAME=${archive##*/} + else + python3 "${FIXTURE_HELPER%/*}/release.py" download + export FIXTURE_ARCHIVE_NAME=$(python3 "${FIXTURE_HELPER%/*}/release.py" name) + fi python3 "$FIXTURE_HELPER" create fixture_verify mkdir -p "$cache_dir" cp "$FIXTURE_DIR/server.tar.gz" "$FIXTURE_DIR/manifest.json" "$cache_dir/" - rm -rf "$source_dir" + if [[ -n "${source_dir:-}" ]]; then rm -rf "$source_dir"; fi + if [[ -n "$candidate_repo" ]]; then rm -rf "$candidate_repo"; fi } fixture_copy_archive() { fixture_validate_java diff --git a/.github/actions/setup-hugegraph-server/manifest.py b/.github/actions/setup-hugegraph-server/manifest.py index c7446cecf..0b13a7514 100644 --- a/.github/actions/setup-hugegraph-server/manifest.py +++ b/.github/actions/setup-hugegraph-server/manifest.py @@ -23,14 +23,18 @@ import re import sys +import release + if sys.argv[1] == "inputs": repo = pathlib.Path(__file__).resolve().parents[3] patterns = ( '.github/actions/setup-hugegraph-server/action.yml', '.github/actions/setup-hugegraph-server/fixture.sh', '.github/actions/setup-hugegraph-server/manifest.py', + '.github/actions/setup-hugegraph-server/release.py', '.github/actions/setup-java-env/action.yml', '.github/configs/settings.xml', 'hugegraph-client/assembly/travis/checkout-server.sh', + 'hugegraph-client/assembly/travis/install-candidate-sdk.sh', 'hugegraph-client/assembly/travis/start-hugegraph-servers.sh', 'hugegraph-*/assembly/travis/install-hugegraph-from-source.sh', 'hugegraph-hubble/hubble-dist/assembly/travis/download-hugegraph.sh', @@ -45,6 +49,16 @@ root = pathlib.Path(os.environ["FIXTURE_DIR"]) identity = {key.lower(): os.environ["FIXTURE_" + key] for key in ("REPOSITORY", "COMMIT", "JAVA", "CONFIG")} +try: + if identity["java"] == "11": + identity.update(release.identity(os.environ.get("FIXTURE_RELEASE_VERSION", release.VERSION), + identity["repository"], identity["commit"])) + else: + identity.update(source_kind="candidate-sdk") +except ValueError as error: + sys.exit(str(error)) + + def digest(): h = hashlib.sha256() with (root / "server.tar.gz").open("rb") as stream: @@ -52,6 +66,8 @@ def digest(): h.update(chunk) return h.hexdigest() try: + if identity["java"] == "11": + release.verify(root / "server.tar.gz") if sys.argv[1] == "create": data = dict(identity, archive_name=os.environ["FIXTURE_ARCHIVE_NAME"], sha256=digest()) (root / "manifest.json").write_text(json.dumps(data, sort_keys=True) + "\n") @@ -63,6 +79,8 @@ def digest(): raise ValueError("fixture origin, commit, JDK or build configuration mismatch") if not re.fullmatch(r"apache-hugegraph-[A-Za-z0-9_.-]+\.tar\.gz", data.get("archive_name", "")): raise ValueError("invalid archive name") + if identity["java"] == "11" and data.get("archive_name") != release.ARCHIVE_NAME: + raise ValueError("official Server release archive name mismatch") if data.get("sha256") != digest(): raise ValueError("fixture checksum mismatch") if sys.argv[1] == "name": diff --git a/.github/actions/setup-hugegraph-server/release.py b/.github/actions/setup-hugegraph-server/release.py new file mode 100644 index 000000000..df42792d1 --- /dev/null +++ b/.github/actions/setup-hugegraph-server/release.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to You under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +"""Pinned ASF binary release identity and bounded download, without source fallback.""" +import hashlib +import os +import pathlib +import subprocess +import sys +import tempfile + +VERSION = "1.7.0" +REPOSITORY = "apache/hugegraph" +COMMIT = "b12425c2032bf0d21a97b8221f42a18055c2982f" +ARCHIVE_NAME = "apache-hugegraph-incubating-1.7.0.tar.gz" +SOURCE_URL = "https://downloads.apache.org/hugegraph/1.7.0/" + ARCHIVE_NAME +ARCHIVE_URL = "https://archive.apache.org/dist/hugegraph/1.7.0/" + ARCHIVE_NAME +# Verified against SOURCE_URL + ".sha512" in the official ASF distribution. +SHA512 = ("ba093203e817f17582895ff10ceb0458498c886ec7a687fd0f3e5f56ba739454" + "ec928a592bdad8a3685651dc9f8ef904d082f9530d4aae6a83b5d9d9626ad13f") + + +def identity(version, repository=REPOSITORY, commit=COMMIT): + if version != VERSION: + raise ValueError("unsupported Server release version: " + version) + if repository != REPOSITORY or commit != COMMIT: + raise ValueError("official Server release requires " + REPOSITORY + "@" + COMMIT) + return dict(source_kind="asf-release", release_version=VERSION, + source_repository=REPOSITORY, source_commit=COMMIT, + source_url=SOURCE_URL, official_sha512=SHA512) + + +def digest(path): + result = hashlib.sha512() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + result.update(chunk) + return result.hexdigest() + + +def verify(path): + if not path.is_file() or path.is_symlink() or digest(path) != SHA512: + raise ValueError("official Server release SHA-512 mismatch") + + +def download(directory): + directory.mkdir(parents=True, exist_ok=True) + descriptor, temporary = tempfile.mkstemp(prefix=".server-release-", dir=directory) + os.close(descriptor) + path = pathlib.Path(temporary) + try: + command = ["curl", "--fail", "--location", "--show-error", "--silent", + "--connect-timeout", "20", "--max-time", "900", "--retry", "2", + "--proto", "=https", "--proto-redir", "=https", + "--output", str(path)] + try: + subprocess.run(command + [SOURCE_URL], check=True) + except subprocess.CalledProcessError: + # Superseded releases leave downloads but remain in the ASF archive. + # The manifest identity stays canonical regardless of transport. + print("ASF distribution download failed; trying the official archive", file=sys.stderr) + subprocess.run(command + [ARCHIVE_URL], check=True) + # A checksum mismatch is fatal; never hide it by trying another source. + verify(path) + path.replace(directory / "server.tar.gz") + finally: + path.unlink(missing_ok=True) + + +if __name__ == "__main__": + try: + identity(os.environ.get("FIXTURE_RELEASE_VERSION", VERSION), + os.environ["FIXTURE_REPOSITORY"], os.environ["FIXTURE_COMMIT"]) + if sys.argv[1] == "validate": + pass + elif sys.argv[1] == "download": + download(pathlib.Path(os.environ["FIXTURE_DIR"])) + elif sys.argv[1] == "name": + print(ARCHIVE_NAME) + else: + raise ValueError("invalid release helper command") + except (OSError, ValueError, KeyError, subprocess.CalledProcessError) as error: + sys.exit(str(error)) diff --git a/.github/actions/setup-hugegraph-server/runtime_test.py b/.github/actions/setup-hugegraph-server/runtime_test.py index 91ce54282..024d2b79a 100644 --- a/.github/actions/setup-hugegraph-server/runtime_test.py +++ b/.github/actions/setup-hugegraph-server/runtime_test.py @@ -18,21 +18,36 @@ """Exercise fixture identity, bounded downloads and service readiness offline.""" import hashlib import json +import importlib.util import os from pathlib import Path import subprocess import tempfile +import shutil import unittest HERE = Path(__file__).resolve().parent ROOT = HERE.parents[2] -SHA = 'a' * 40 +SHA = 'b12425c2032bf0d21a97b8221f42a18055c2982f' class RuntimeTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() self.addCleanup(self.temp.cleanup) self.dir = Path(self.temp.name) + # Offline archives use a test-local checksum pin, never a production bypass. + self.action = self.dir / 'action' + self.action.mkdir() + for name in ('fixture.sh', 'manifest.py', 'release.py', 'service-wait.sh'): + shutil.copyfile(HERE / name, self.action / name) + spec = importlib.util.spec_from_file_location('fixture_release', HERE / 'release.py') + self.release = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.release) + checksum = hashlib.sha512(b'archive').hexdigest() + release_code = (self.action / 'release.py').read_text() + release_code = release_code.replace(self.release.SHA512[:64], checksum[:64]) + release_code = release_code.replace(self.release.SHA512[64:], checksum[64:]) + (self.action / 'release.py').write_text(release_code) self.bin = self.dir / 'bin' self.bin.mkdir() (self.bin/'java').write_text("#!/bin/bash\necho 'openjdk version \"11.0.1\"' >&2\n") @@ -41,7 +56,8 @@ def setUp(self): FIXTURE_DIR=str(self.dir / 'fixture'), FIXTURE_REPOSITORY='apache/hugegraph', FIXTURE_COMMIT=SHA, FIXTURE_JAVA='11', FIXTURE_BUILD_INPUTS='test-build-contract', FIXTURE_PLATFORM='Linux-X64', - FIXTURE_CONFIG='package -DskipTests -Dmaven.javadoc.skip=true -ntp;schema=1;inputs=test-build-contract;platform=Linux-X64') + FIXTURE_RELEASE_VERSION='1.7.0', + FIXTURE_CONFIG='asf-release-1.7.0;schema=2;inputs=test-build-contract;platform=Linux-X64') Path(self.env['FIXTURE_DIR']).mkdir() def command(self, name, body): script = self.bin / name @@ -54,27 +70,37 @@ def manifest(self, **extra): root = Path(self.env['FIXTURE_DIR']) data = {k.lower(): self.env['FIXTURE_'+k] for k in ('REPOSITORY','COMMIT','JAVA','CONFIG')} (root/'server.tar.gz').write_bytes(b'archive') - data.update(archive_name='apache-hugegraph-fixture.tar.gz', sha256=hashlib.sha256(b'archive').hexdigest()) + if extra.get('java', self.env['FIXTURE_JAVA']) == '11': + data.update(self.release.identity('1.7.0')) + data['official_sha512'] = hashlib.sha512(b'archive').hexdigest() + name = self.release.ARCHIVE_NAME + else: + data.update(source_kind='candidate-sdk') + name = 'apache-hugegraph-fixture.tar.gz' + data.update(archive_name=name, sha256=hashlib.sha256(b'archive').hexdigest()) data.update(extra) (root/'manifest.json').write_text(json.dumps(data)) def test_identity_checksum_and_regular_files(self): self.manifest() env = dict(self.env, FIXTURE_MODE='start') - self.assertEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0) - for field, wrong in [('repository','fork/hugegraph'), ('commit','b'*40), ('java','17'), ('config','different'), ('sha256','bad')]: + self.assertEqual(self.run_script(self.action/'fixture.sh', env=env).returncode, 0) + for field, wrong in [('repository','fork/hugegraph'), ('commit','b'*40), ('java','17'), + ('config','different'), ('source_url','https://example.invalid/release'), + ('release_version','1.8.0'), ('source_repository','fork/hugegraph'), + ('source_commit','b'*40), ('official_sha512','bad'), ('sha256','bad')]: self.manifest(**{field:wrong}) - self.assertNotEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0, field) + self.assertNotEqual(self.run_script(self.action/'fixture.sh', env=env).returncode, 0, field) self.manifest() self.command('java', "echo 'openjdk version \"17.0.1\"' >&2\n") - self.assertNotEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0) + self.assertNotEqual(self.run_script(self.action/'fixture.sh', env=env).returncode, 0) self.command('java', "echo 'openjdk version \"11.0.1\"' >&2\n") self.manifest(archive_name='../escape.tar.gz') - self.assertNotEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0) + self.assertNotEqual(self.run_script(self.action/'fixture.sh', env=env).returncode, 0) self.manifest() archive = Path(self.env['FIXTURE_DIR'])/'server.tar.gz' archive.unlink() archive.symlink_to(self.dir/'missing') - self.assertNotEqual(self.run_script(HERE/'fixture.sh', env=env).returncode, 0) + self.assertNotEqual(self.run_script(self.action/'fixture.sh', env=env).returncode, 0) def test_composite_step_isolates_fixture_jdk_from_caller(self): action = (HERE / 'action.yml').read_text() block = action.split(' run: |\n', 1)[1].split(' - name:', 1)[0] @@ -96,11 +122,14 @@ def test_composite_step_isolates_fixture_jdk_from_caller(self): for fixture_version, fixture_jdk, caller_version, caller_jdk in ( (11, jdk11, 17, jdk17), (17, jdk17, 11, jdk11)): with self.subTest(fixture=fixture_version, caller=caller_version): - self.manifest(java=str(fixture_version)) + config = self.env['FIXTURE_CONFIG'] + if fixture_version == 17: + config = 'candidate-sdk-install;' + config.split(';', 1)[1] + self.manifest(java=str(fixture_version), config=config) env = dict(self.env, JAVA_HOME=str(caller_jdk), PATH=f'{caller_jdk}/bin:{self.env["PATH"]}', FIXTURE_JAVA=str(fixture_version), FIXTURE_JAVA_HOME=str(fixture_jdk), - FIXTURE_MODE='start', FIXTURE_ARTIFACT='shared', FIXTURE_ACTION=str(HERE), + FIXTURE_MODE='start', FIXTURE_ARTIFACT='shared', FIXTURE_ACTION=str(self.action), ISOLATE_FIXTURE_REPO='false', TRAVIS_DIR=str(travis), RUNNER_TEMP=str(self.dir)) result = subprocess.run( ['bash', '-c', 'bash "$HOME/action-step.sh" && java -version 2> "$HOME/caller-java"'], @@ -197,15 +226,121 @@ def test_exact_cache_hit_never_builds_or_starts(self): (Path(self.env['FIXTURE_DIR'])/name).unlink() self.command('git', 'echo forbidden-git >> "$HOME/forbidden"; exit 99\n') self.command('mvn', 'echo forbidden-build >> "$HOME/forbidden"; exit 99\n') - result = self.run_script(HERE/'fixture.sh', env=dict(self.env, FIXTURE_MODE='build')) + self.command('curl', 'echo download >> "$HOME/downloads"; exit 22\n') + result = self.run_script(self.action/'fixture.sh', env=dict(self.env, FIXTURE_MODE='build')) self.assertEqual(result.returncode, 0, result.stderr) self.assertFalse((self.dir/'forbidden').exists()) - self.assertFalse((self.dir/'apache-hugegraph-fixture.tar.gz').exists()) - # An incomplete exact cache must rebuild, rather than poison future runs. + self.assertFalse((self.dir/self.release.ARCHIVE_NAME).exists()) + # An incomplete release cache must download again, never rebuild old source. (cache/'manifest.json').unlink() - result = self.run_script(HERE/'fixture.sh', env=dict(self.env, FIXTURE_MODE='build')) + result = self.run_script(self.action/'fixture.sh', env=dict(self.env, FIXTURE_MODE='build')) self.assertNotEqual(result.returncode, 0) - self.assertTrue((self.dir/'forbidden').exists()) + self.assertFalse((self.dir/'forbidden').exists()) + self.assertTrue((self.dir/'downloads').exists()) + + def test_release_pin_matches_official_identity_and_rejects_wrong_bytes(self): + self.assertEqual('1.7.0', self.release.VERSION) + self.assertEqual('https://downloads.apache.org/hugegraph/1.7.0/' + + self.release.ARCHIVE_NAME, self.release.SOURCE_URL) + self.assertEqual('apache/hugegraph', self.release.REPOSITORY) + self.assertEqual(SHA, self.release.COMMIT) + self.assertEqual(128, len(self.release.SHA512)) + self.assertEqual(self.release.SHA512, + self.release.identity('1.7.0')['official_sha512']) + with self.assertRaises(ValueError): + self.release.identity('1.8.0') + archive = self.dir / 'invalid-release.tar.gz' + archive.write_bytes(b'archive') + with self.assertRaisesRegex(ValueError, 'SHA-512 mismatch'): + self.release.verify(archive) + + def test_release_rejects_wrong_caller_source_before_cache_or_download(self): + for name in ('curl', 'git', 'mvn'): + self.command(name, 'echo forbidden >> "$HOME/forbidden"; exit 99\n') + for field, wrong in (('FIXTURE_REPOSITORY', 'fork/hugegraph'), + ('FIXTURE_COMMIT', 'a' * 40)): + with self.subTest(field=field): + env = dict(self.env, FIXTURE_MODE='build', **{field: wrong}) + result = self.run_script(self.action / 'fixture.sh', env=env) + self.assertNotEqual(0, result.returncode) + self.assertIn('official Server release requires', result.stderr) + self.assertFalse((self.dir / 'forbidden').exists()) + self.assertFalse((self.dir / 'hugegraph-fixture-cache').exists()) + self.assertFalse((Path(self.env['FIXTURE_DIR']) / 'manifest.json').exists()) + # Direct manifest callers must reject the same false source claim. + result = subprocess.run(['python3', str(self.action / 'manifest.py'), 'create'], + env=env, text=True, capture_output=True, timeout=20) + self.assertNotEqual(0, result.returncode) + self.assertIn('official Server release requires', result.stderr) + self.assertFalse((Path(self.env['FIXTURE_DIR']) / 'manifest.json').exists()) + + def test_release_download_keeps_official_identity_and_never_builds(self): + self.command('curl', 'printf "%s\\n" "$@" > "$HOME/download-args"\n' + 'while [[ $# -gt 0 ]]; do\n' + ' if [[ "$1" == --output ]]; then printf archive > "$2"; exit 0; fi\n' + ' shift\ndone\nexit 1\n') + for name in ('git', 'mvn'): + self.command(name, 'echo forbidden >> "$HOME/forbidden"; exit 99\n') + result = self.run_script(self.action / 'fixture.sh', env=dict(self.env, FIXTURE_MODE='build')) + self.assertEqual(0, result.returncode, result.stderr) + self.assertFalse((self.dir / 'forbidden').exists()) + manifest = json.loads((Path(self.env['FIXTURE_DIR']) / 'manifest.json').read_text()) + self.assertEqual('asf-release', manifest['source_kind']) + self.assertEqual(self.release.REPOSITORY, manifest['source_repository']) + self.assertEqual(self.release.COMMIT, manifest['source_commit']) + self.assertEqual(self.release.SOURCE_URL, manifest['source_url']) + self.assertEqual(hashlib.sha512(b'archive').hexdigest(), manifest['official_sha512']) + args = (self.dir / 'download-args').read_text() + self.assertIn('--max-time\n900', args) + self.assertIn('--proto\n=https', args) + self.assertIn(self.release.SOURCE_URL, args) + + def test_release_archive_fallback_keeps_canonical_identity(self): + self.command('curl', 'printf "%s\\n" "$@" >> "$HOME/download-args"\n' + 'if [[ ! -f "$HOME/primary-failed" ]]; then\n' + ' touch "$HOME/primary-failed"; exit 22\nfi\n' + 'while [[ $# -gt 0 ]]; do\n' + ' if [[ "$1" == --output ]]; then printf archive > "$2"; exit 0; fi\n' + ' shift\ndone\nexit 1\n') + result = self.run_script(self.action / 'fixture.sh', env=dict(self.env, FIXTURE_MODE='build')) + self.assertEqual(0, result.returncode, result.stderr) + args = (self.dir / 'download-args').read_text() + self.assertLess(args.index(self.release.SOURCE_URL), args.index(self.release.ARCHIVE_URL)) + manifest = json.loads((Path(self.env['FIXTURE_DIR']) / 'manifest.json').read_text()) + self.assertEqual(self.release.SOURCE_URL, manifest['source_url']) + self.assertEqual(hashlib.sha512(b'archive').hexdigest(), manifest['official_sha512']) + + def test_release_bad_checksum_does_not_try_another_source(self): + self.command('curl', 'printf "%s\\n" "$@" >> "$HOME/download-args"\n' + 'while [[ $# -gt 0 ]]; do\n' + ' if [[ "$1" == --output ]]; then printf tampered > "$2"; exit 0; fi\n' + ' shift\ndone\nexit 1\n') + result = self.run_script(self.action / 'fixture.sh', env=dict(self.env, FIXTURE_MODE='build')) + self.assertNotEqual(0, result.returncode) + self.assertIn('SHA-512 mismatch', result.stderr) + self.assertNotIn(self.release.ARCHIVE_URL, (self.dir / 'download-args').read_text()) + self.assertFalse((Path(self.env['FIXTURE_DIR']) / 'server.tar.gz').exists()) + + def test_release_failure_never_falls_back_to_source_or_publishes_cache(self): + self.command('curl', 'echo download >> "$HOME/downloads"; exit 22\n') + for name in ('git', 'mvn'): + self.command(name, 'echo forbidden >> "$HOME/forbidden"; exit 99\n') + result = self.run_script(self.action / 'fixture.sh', env=dict(self.env, FIXTURE_MODE='build')) + self.assertNotEqual(0, result.returncode) + self.assertFalse((self.dir / 'forbidden').exists()) + self.assertFalse((self.dir / 'hugegraph-fixture-cache').exists()) + self.assertEqual([], list(Path(self.env['FIXTURE_DIR']).iterdir())) + + def test_release_rejects_tampering_even_with_updated_local_sha256(self): + self.manifest() + fixture = Path(self.env['FIXTURE_DIR']) + (fixture / 'server.tar.gz').write_bytes(b'changed archive') + manifest = json.loads((fixture / 'manifest.json').read_text()) + manifest['sha256'] = hashlib.sha256(b'changed archive').hexdigest() + (fixture / 'manifest.json').write_text(json.dumps(manifest)) + result = self.run_script(self.action / 'fixture.sh', env=dict(self.env, FIXTURE_MODE='start')) + self.assertNotEqual(0, result.returncode) + self.assertIn('official Server release SHA-512 mismatch', result.stderr) if __name__ == '__main__': diff --git a/.github/actions/setup-java-env/action.yml b/.github/actions/setup-java-env/action.yml index 517546cd8..f84369f52 100644 --- a/.github/actions/setup-java-env/action.yml +++ b/.github/actions/setup-java-env/action.yml @@ -23,12 +23,20 @@ runs: cache-dependency-path: | **/pom.xml .github/configs/settings.xml + .github/actions/setup-candidate-sdk/action.yml overwrite-settings: false - name: Configure Maven repositories shell: bash run: | set -euo pipefail + # Reuse third-party dependencies, never a previously installed HugeGraph SDK. + rm -rf -- "$HOME/.m2/repository/org/apache/hugegraph" + rm -f -- "$HOME/.m2/repository/candidate-sdk-manifest.json" + # Cached failed downloads must not suppress checking a recovered repository. + if [[ -d "$HOME/.m2/repository" ]]; then + find "$HOME/.m2/repository" -type f -name '*.lastUpdated' -delete + fi maven_version=$(mvn -version | sed $'s/\033\\[[0-9;]*m//g' | sed -n 's/^Apache Maven \([0-9.]*\).*/\1/p' | head -1) if [[ ! "$maven_version" =~ ^([0-9]+)\.([0-9]+)\. ]] || (( 10#${BASH_REMATCH[1]:-0} < 3 || diff --git a/.github/actions/upload-coverage/action.yml b/.github/actions/upload-coverage/action.yml index bf8ba8cd8..bedc40b08 100644 --- a/.github/actions/upload-coverage/action.yml +++ b/.github/actions/upload-coverage/action.yml @@ -3,8 +3,8 @@ description: 'Upload test coverage report to Codecov' inputs: token: - description: 'Codecov upload token' - required: true + description: 'Legacy compatibility input; OIDC authenticates uploads instead' + required: false file: description: 'Path to the coverage report file' required: false @@ -16,6 +16,8 @@ runs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v7 with: - token: ${{ inputs.token }} + use_oidc: true + disable_search: true + fail_ci_if_error: false slug: ${{ github.repository }} files: ${{ inputs.file }} diff --git a/.github/scripts/test_candidate_sdk.py b/.github/scripts/test_candidate_sdk.py index 7d27fc899..1621defeb 100644 --- a/.github/scripts/test_candidate_sdk.py +++ b/.github/scripts/test_candidate_sdk.py @@ -20,6 +20,7 @@ import copy import json import os +from unittest.mock import patch from pathlib import Path import shutil import subprocess @@ -43,7 +44,7 @@ def test_license_metadata_can_rebuild_installed_root_parent(self): settings.write_text("") header = '4.0.0' flatten = ('org.codehaus.mojoflatten-maven-plugin' - '1.2.7true' + '1.3.0true' 'resolveCiFriendliesOnly' 'process-resourcesflatten' '') @@ -78,7 +79,7 @@ def run(*arguments): "-Dmaven.repo.local=" + str(root / "m2"), "-B", "-ntp", *arguments], cwd=source, capture_output=True, text=True, timeout=240) - goals = (["org.codehaus.mojo:flatten-maven-plugin:1.2.7:flatten", "install", + goals = (["org.codehaus.mojo:flatten-maven-plugin:1.3.0:flatten", "install", "-Dflatten.mode=resolveCiFriendliesOnly", "-DupdatePomFile=true"] if fixed else ["install"]) literal_request = group + ":root:pom:${revision}" @@ -106,6 +107,12 @@ def run(*arguments): class CandidateDistributionTest(unittest.TestCase): + def setUp(self): + self.source_commit = sdk.COMMIT + context = patch.dict(os.environ, {"CANDIDATE_SOURCE_COMMIT": self.source_commit}) + context.start() + self.addCleanup(context.stop) + def fixture(self, root): repository = root / "m2" artifacts, modules = [], [] @@ -126,7 +133,7 @@ def fixture(self, root): modules.append({"group_id": "org.apache.hugegraph", "artifact_id": artifact, "version": "1.7.0", "packaging": packaging, "source_pom": source_pom, "files": files}) - manifest = {"repository": sdk.REPOSITORY, "commit": sdk.COMMIT, + manifest = {"repository": sdk.REPOSITORY, "commit": self.source_commit, "source_revision": "1.7.0", "java_version": "17", "required_sdk_modules": modules, "artifacts": artifacts} self.write_manifest(repository, manifest) @@ -260,6 +267,53 @@ def test_reactor_installs_do_not_change_the_locked_sdk_contract(self): with self.assertRaisesRegex(RuntimeError, "SDK artifact hash mismatch"): sdk.validate_distribution(repository, directory, "hubble") + def test_source_context_preserves_full_identity_checks(self): + with tempfile.TemporaryDirectory() as temporary: + repository, manifest = self.fixture(Path(temporary)) + manifest["commit"] = sdk.COMMIT[:6] + "b" * 34 + self.write_manifest(repository, manifest) + with self.assertRaisesRegex(RuntimeError, "SDK source does not match"): + sdk.validate_sdk(repository) + with patch.dict(os.environ, {"CANDIDATE_SOURCE_COMMIT": sdk.COMMIT[:6]}): + with self.assertRaisesRegex(RuntimeError, "Invalid candidate SDK manifest"): + sdk.validate_sdk(repository) + + def test_explicit_resolved_context_supports_latest_master_provenance(self): + with tempfile.TemporaryDirectory() as temporary: + repository, manifest = self.fixture(Path(temporary)) + selected = "b" * 40 + manifest["commit"] = selected + self.write_manifest(repository, manifest) + with patch.dict(os.environ, {"CANDIDATE_SOURCE_COMMIT": selected}): + sdk.validate_sdk(repository) + with self.assertRaisesRegex(RuntimeError, "SDK source does not match"): + sdk.validate_sdk(repository) + + def test_standalone_packaging_uses_the_local_lock(self): + with tempfile.TemporaryDirectory() as temporary: + repository, _ = self.fixture(Path(temporary)) + with patch.dict(os.environ, {}, clear=True): + sdk.validate_sdk(repository) + + def test_candidate_manifest_requires_the_supported_build_jdk(self): + with tempfile.TemporaryDirectory() as temporary: + repository, manifest = self.fixture(Path(temporary)) + for version in ("11", "18", "21"): + with self.subTest(version=version): + manifest["java_version"] = version + self.write_manifest(repository, manifest) + with self.assertRaisesRegex(RuntimeError, "Java version"): + sdk.validate_sdk(repository) + for version in (17.9, 17, True, "17.0"): + with self.subTest(version=version): + manifest["java_version"] = version + self.write_manifest(repository, manifest) + with self.assertRaisesRegex(RuntimeError, "Invalid candidate SDK manifest"): + sdk.validate_sdk(repository) + manifest["java_version"] = "17" + self.write_manifest(repository, manifest) + sdk.validate_sdk(repository) + def test_malformed_manifests_have_controlled_diagnostics(self): with tempfile.TemporaryDirectory() as temporary: repository, manifest = self.fixture(Path(temporary)) diff --git a/.github/scripts/verify_candidate_image_sdk.py b/.github/scripts/verify_candidate_image_sdk.py index f2ec28090..291ae46d6 100644 --- a/.github/scripts/verify_candidate_image_sdk.py +++ b/.github/scripts/verify_candidate_image_sdk.py @@ -20,10 +20,12 @@ import argparse import hashlib import json +import os +import re from pathlib import Path REPOSITORY = "apache/hugegraph" -COMMIT = "d9abcd4317fb36128e7e4d209139ec7f3a28cdfc" +COMMIT = "e62c961e00221569d4f955abbadf60faee45b283" REQUIRED_MODULES = { "pom.xml", "hugegraph-commons/pom.xml", "hugegraph-server/pom.xml", "hugegraph-pd/pom.xml", "hugegraph-store/pom.xml", @@ -51,6 +53,14 @@ def digest(path): return checksum.hexdigest() +def source_commit(): + # The SDK action exports the full identity already verified by the source resolver. + expected = os.environ.get("CANDIDATE_SOURCE_COMMIT", COMMIT) + if not isinstance(expected, str) or not re.fullmatch(r"[0-9a-f]{40}", expected): + raise ValueError("Invalid resolved Server source identity") + return expected + + def validate_sdk(repository): try: return _validate_sdk(Path(repository)) @@ -62,9 +72,12 @@ def validate_sdk(repository): def _validate_sdk(repository): repository = Path(repository) manifest = json.loads((repository / "candidate-sdk-manifest.json").read_text()) - if (manifest["repository"], manifest["commit"]) != (REPOSITORY, COMMIT): + if (manifest["repository"], manifest["commit"]) != (REPOSITORY, source_commit()): raise RuntimeError("SDK source does not match the locked candidate") - if manifest["source_revision"] != "1.7.0" or int(manifest["java_version"]) < 17: + java_version = manifest["java_version"] + if not isinstance(java_version, str) or not java_version.isdecimal(): + raise ValueError("Invalid Java version format") + if manifest["source_revision"] != "1.7.0" or java_version != "17": raise RuntimeError("SDK revision or Java version does not match the candidate contract") modules = manifest["required_sdk_modules"] if len(modules) != len(REQUIRED_MODULES) or {m["source_pom"] for m in modules} != REQUIRED_MODULES: @@ -161,4 +174,4 @@ def validate_distribution(repository, directory, module): validate_distribution(args.repository, args.distribution, args.module) except RuntimeError as error: parser.exit(1, str(error) + "\n") - print(f"Verified {manifest['repository']}@{manifest['commit']}: {len(jars)} candidate JARs") + print(f"Verified {manifest['repository']}@{manifest['commit'][:6]}: {len(jars)} candidate JARs") diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 784e75ae1..1813c0214 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,8 +86,8 @@ jobs: uses: ./.github/actions/get-hugegraph-commit with: repository: apache/hugegraph - ref: d9abcd4317fb36128e7e4d209139ec7f3a28cdfc - expected-commit: d9abcd4317fb36128e7e4d209139ec7f3a28cdfc + ref: e62c961e00221569d4f955abbadf60faee45b283 + expected-commit: e62c961e00221569d4f955abbadf60faee45b283 - name: Record external test inputs env: NEEDS_SERVER: ${{ steps.scope.outputs.needsFixture }} @@ -192,6 +192,10 @@ jobs: java-version: '17' client: + permissions: + contents: read + actions: read + id-token: write needs: [plan, fixture] if: ${{ needs.plan.outputs.client == 'true' }} uses: ./.github/workflows/client-ci.yml @@ -203,6 +207,10 @@ jobs: secrets: inherit loader: + permissions: + contents: read + actions: read + id-token: write needs: [plan, fixture] if: ${{ needs.plan.outputs.loader == 'true' }} uses: ./.github/workflows/loader-ci.yml @@ -236,6 +244,10 @@ jobs: secrets: inherit hubble: + permissions: + contents: read + actions: read + id-token: write needs: [plan, fixture, hubble-fixture] if: ${{ needs.plan.outputs.hubble == 'true' }} uses: ./.github/workflows/hubble-ci.yml diff --git a/.github/workflows/client-ci.yml b/.github/workflows/client-ci.yml index f0f1f32f2..017644c49 100644 --- a/.github/workflows/client-ci.yml +++ b/.github/workflows/client-ci.yml @@ -83,6 +83,7 @@ jobs: mvn test -Dtest=FuncTestSuite - name: Upload coverage to Codecov + continue-on-error: true uses: ./.github/actions/upload-coverage with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/hubble-ci.yml b/.github/workflows/hubble-ci.yml index 0ac67b871..c43242cd8 100644 --- a/.github/workflows/hubble-ci.yml +++ b/.github/workflows/hubble-ci.yml @@ -87,15 +87,6 @@ jobs: with: node-version: '18.20.8' - # we also should cache python & yarn & downloads to avoid useless work - - name: Cache Maven packages - uses: actions/cache@v6 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - name: Cache Node modules uses: actions/cache@v6 with: @@ -249,6 +240,7 @@ jobs: retention-days: 14 - name: Upload coverage to Codecov + continue-on-error: true uses: ./.github/actions/upload-coverage with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/loader-ci.yml b/.github/workflows/loader-ci.yml index 676dd81a4..87e16a5a3 100644 --- a/.github/workflows/loader-ci.yml +++ b/.github/workflows/loader-ci.yml @@ -110,6 +110,7 @@ jobs: done - name: Upload coverage to Codecov + continue-on-error: true uses: ./.github/actions/upload-coverage with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/upstream-compatibility.yml b/.github/workflows/upstream-compatibility.yml new file mode 100644 index 000000000..524b92971 --- /dev/null +++ b/.github/workflows/upstream-compatibility.yml @@ -0,0 +1,85 @@ +name: Upstream compatibility + +on: + schedule: + - cron: '17 3 * * *' + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/upstream-compatibility.yml' + - '.github/actions/get-hugegraph-commit/**' + - '.github/actions/setup-candidate-sdk/**' + - 'hugegraph-client/assembly/travis/install-candidate-sdk.sh' + - 'hugegraph-client/assembly/travis/checkout-server.sh' + - '.github/scripts/verify_candidate_image_sdk.py' + +permissions: + contents: read + +concurrency: + group: upstream-compatibility-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + upstream-master: + if: github.event_name != 'schedule' || github.repository == 'apache/hugegraph-toolchain' + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Fetch code + uses: actions/checkout@v7 + + - name: Setup Java environment + uses: ./.github/actions/setup-java-env + with: + java-version: '17' + distribution: temurin + + - name: Resolve latest ASF master once + id: upstream + uses: ./.github/actions/get-hugegraph-commit + with: + repository: apache/hugegraph + ref: master + + - name: Install SDK from the resolved commit + id: sdk + uses: ./.github/actions/setup-candidate-sdk + with: + repository: apache/hugegraph + ref: ${{ steps.upstream.outputs.commit_id }} + expected-commit: ${{ steps.upstream.outputs.commit_id }} + + - name: Compile core consumers + run: | + mvn install -pl hugegraph-client,hugegraph-loader,hugegraph-tools,hugegraph-spark-connector -am -DskipTests -Dmaven.javadoc.skip=true -ntp + mvn compile -pl hugegraph-hubble/hubble-be -Dmaven.javadoc.skip=true -ntp + + - name: Start the same-source Server fixture + env: + CANDIDATE_ARCHIVE: ${{ steps.sdk.outputs.archive }} + run: bash hugegraph-client/assembly/travis/start-hugegraph-servers.sh "$CANDIDATE_ARCHIVE" + + - name: Client unit and HTTP compatibility tests + working-directory: hugegraph-client + run: | + mvn test -Dtest=UnitTestSuite -ntp + mvn test -Dtest=ApiTestSuite -ntp + mvn test -Dtest=FuncTestSuite -ntp + + - name: Loader and Hubble backend unit tests + run: | + mvn test -P unit -pl hugegraph-loader -ntp + mvn test -P unit-test -pl hugegraph-hubble/hubble-be -ntp + + - name: Tools functional compatibility tests + run: mvn test -Dtest=FuncTestSuite -Dtools.test.disposable=true -pl hugegraph-tools -ntp + + - name: Retain test reports + if: always() + uses: actions/upload-artifact@v7 + with: + name: upstream-compatibility-reports-${{ github.run_attempt }} + path: '**/target/surefire-reports/**' + if-no-files-found: warn + retention-days: 7 diff --git a/AGENTS.md b/AGENTS.md index 9f8ca9eb7..abce8c1d2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -132,10 +132,12 @@ hugegraph-loader, hugegraph-tools, hugegraph-hubble, hugegraph-spark-connector ## Docker +Run from the Toolchain repository root; the Dockerfiles copy shared SDK helpers. + ```bash # Loader -cd hugegraph-loader && docker build -t hugegraph/hugegraph-loader:latest . +docker build -f hugegraph-loader/Dockerfile -t hugegraph/hugegraph-loader:latest . # Hubble -cd hugegraph-hubble && docker build -t hugegraph/hugegraph-hubble:latest . +docker build -f hugegraph-hubble/Dockerfile -t hugegraph/hugegraph-hubble:latest . ``` diff --git a/README.md b/README.md index b2bad81a0..a40b242c2 100644 --- a/README.md +++ b/README.md @@ -433,7 +433,7 @@ docker run --rm \ ./bin/hugegraph-loader.sh -f /config/mapping.json ``` -Build images locally: +Build images from the Toolchain repository root so the context contains the shared SDK bootstrap and verifier: ```bash # Loader docker build -f hugegraph-loader/Dockerfile \ diff --git a/hugegraph-client/assembly/travis/codecov.yml b/codecov.yml similarity index 80% rename from hugegraph-client/assembly/travis/codecov.yml rename to codecov.yml index 106e68f94..9d4c85f1f 100644 --- a/hugegraph-client/assembly/travis/codecov.yml +++ b/codecov.yml @@ -14,5 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # +coverage: + status: + project: + default: + informational: true + patch: + default: + informational: true + ignore: - - "src/main/java/org/apache/hugegraph/example" + - "hugegraph-client/src/main/java/org/apache/hugegraph/example" diff --git a/docs/ci.md b/docs/ci.md index 5b826a783..1bc6897e0 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -1,8 +1,6 @@ # Continuous integration -Only `check-license-header` (headers and RAT) is required for merging. Execution -selection remains conservative: changed modules and their consumers run, but their -results are advisory. Maintainers decide whether to rerun or request changes. +Only `check-license-header` (headers and RAT) is required for merging. Execution selection remains conservative: changed modules and their consumers run, but their results are advisory. Maintainers decide whether to rerun or request changes. The independent workflows are: @@ -12,11 +10,7 @@ The independent workflows are: | Module validation | Consumers, images, fixtures and dependency inventory audit | No | Can be cancelled without cancelling license checks or CodeQL | | CodeQL | Affected-input scans and dependency review; weekly full scan | No | Independent of module tests | -`affected-module-tests` remains a truthful advisory summary under its existing -name for migration. It fails when a selected test, image or fixture fails, is -cancelled or unexpectedly skipped; it does not make these checks required. -Plan and result summaries show specific checks, triggering paths, required roles -and actual results. Native job checks provide the detailed logs. +`affected-module-tests` remains a truthful advisory summary under its existing name for migration. It fails when a selected test, image or fixture fails, is cancelled or unexpectedly skipped; it does not make these checks required. Plan and result summaries show specific checks, triggering paths, required roles and actual results. Native job checks provide the detailed logs. | Change | Tests | | --- | --- | @@ -27,8 +21,7 @@ and actual results. Native job checks provide the detailed logs. | Shared build inputs or unrecognized paths | All modules | | One workflow | Its tests and shared dependencies | -Image selection has its own matrix and does not follow the Client → Loader → -Hubble module-test dependency closure: +Image selection has its own matrix and does not follow the Client → Loader → Hubble module-test dependency closure: | Image input change | Images built and started | | --- | --- | @@ -39,116 +32,35 @@ Hubble module-test dependency closure: | Root/Client/reactor POMs, `.mvn`, `.dockerignore`, shared release docs or unknown shared inputs | Loader and Hubble | | Ordinary Java/frontend application source or tests | None; the module behavior tests above still run | -Dockerfile-only changes run the image check without building Server fixtures. -Hubble's Dockerfile builds Client and Loader before Hubble, so Loader packaging -changes also select Hubble. Packaged README changes select images; the existing -nonempty Hubble README content contract still requires fresh validation if the -file is missing, empty or has an invalid mode. Every selected module and image -runs in the current workflow; successful results from another run are not reused. - -Each image job builds the checkout's actual Dockerfile, records its image ID and -starts a local run-scoped tag with pulling disabled. The container's image ID must -match the build. Loader directly executes `./bin/hugegraph-loader.sh --help`, -including its executable permission and shebang; its default idle -container alone cannot pass. Hubble must remain running and return a valid `/about` -JSON response with the application name and version. Its public root page must -contain the React root and serve nonempty local JavaScript bundles and referenced -stylesheets with the correct content types, rejecting the HTML fallback for missing -assets. These checks validate image build, packaging, startup and public entry -points; Loader data ingestion and Hubble browser workflows remain covered by -module tests. They do not publish an image. - -Selected Client, Loader, Tools, Spark and Hubble tests compile and run on Java 17, -including Loader's separate HDFS jobs. Core module tests cover the locked candidate -and released Server 1.7 where configured; these runtime checks supplement affected-module -selection without adding unrelated modules. Their results remain advisory. - -The historical Server 1.7 fixture runs on its own Java 11 JVM. The locked candidate -Server runs on Java 17 from a verified commit on `apache/hugegraph` master. -The SDK action, CI fixtures, image defaults and packaging verifier use the same -locked source commit. - -The released Server package and Hubble's candidate fixture are each built once and -shared, with independent services per job. Reuse verifies source, commit, JDK, build -inputs and archive checksum. Client, Loader and Tools candidate tests start the -archive already produced by their SDK bootstrap. SDK generation remains job-local -until distinct published SDK coordinates are available. The bootstrap flattens CI-friendly -POM versions before installation, including the root parent, so dependency and license -metadata readers consume concrete coordinates without changing the locked source POMs. -The service JVM is scoped to startup; Toolchain compilation and tests remain on Java 17. - -The shared Java initialization action requires Maven 3.9 or newer and supplies -explicit settings through `MAVEN_ARGS`, preserving existing arguments and user -settings. Dependencies are resolved from Central before ASF Stage, with Stage -snapshots disabled; build plugins use the standard public repositories. Maven cache -keys include the repository settings so changes to the Stage URL invalidate the cache. - -Loader validates its existing profiles against both configured Server baselines. The -Ubuntu HDFS jobs use the ASF Hadoop 3.3.6 image pinned by digest, with separate NameNode -and DataNode containers on the host network. Tests run on Java 17 and use -`localhost:8020`. Startup checks live DataNode registration and a real block -write/read before testing; containers are removed afterward. Image pulling and -readiness are bounded. Cold runners still need to pull the image. - -The immutable Server packages, `ci-plan` and `ci-test-results` diagnostic artifacts -are retained for seven days from their upload. Partial reruns need the original -plan and fixture within that window. After an artifact expires, rerun the whole -workflow to rebuild its plan and fixtures; extending only the result report cannot -restore an expired package. Test reports and coverage artifacts keep their -existing retention settings. - -Documentation paths use an explicit allowlist. A PR containing only plain prose -and static documentation assets has no module tests. Selection uses the cumulative -PR diff, so a documentation update to a PR that also changes source runs its -selected consumers again. Source, type definitions, tests and CI configuration -are never treated as docs. The gate checks the current workflow's actual selected -job results and fixture producers. Its result report records the run, attempt and -input metadata for diagnostics; it is not a reusable success receipt. -The planner determines affected consumers before resolving external Server inputs. -Plans that require Server fixtures record the historical release and locked -candidate source; Go-only module tests consume only the release fixture. Image-only -and plain documentation plans need neither baseline. Unverifiable selection -conservatively resolves both. A failed selected baseline lookup fails planning; -without a published plan, recovery requires a full workflow rerun. - -PR planning records the event's head, base and source before querying live metadata. -A known mismatch with the checkout merge, current head or source fails planning. -The report records the actual tested base and merge; advancing master alone does -not invalidate a completed result, consistent with non-strict branch protection. -Unknown selection falls back to full validation. No cross-run success is reused. - -CodeQL starts independently on affected PR and push inputs, with its original -analysis identity and weekly full scan. Java, frontend and Python changes select -their supported scan language; shared/unknown build inputs keep all three languages. -Go-only and image-only changes do not start unrelated source scans. Manual and -weekly runs scan all three languages. Plain prose-only PRs skip compilation, -fixtures, images and scanning; native workflows retain lightweight selection jobs. -Packaged README files and shared build/configuration inputs are not plain prose. -The `CI_OPTIMIZED_REQUIRED` variable is no longer needed. - -The dependency inventory audit includes root `.mvn/**`, POMs, dependency manifests, -assembly/license inputs and staged Maven settings. It runs from the repository root -with the same root Maven configuration, separately from the required license check. - -For rollout, `.asf.yaml` requests only `check-license-header`. Confirm actual branch -protection has reconciled before relying on advisory cancellation. The existing -module summary remains available during migration. -`Analyze (java)` runs for Java or shared/unknown build inputs, and for manual and -weekly full scans; it is not emitted for every PR. Old required names must not -remain permanently Expected. - -A new PR head cancels older first attempts. Reruns use separate concurrency groups, -so retrying an old commit cannot cancel the current head. Automatic retries run -failed **push** license jobs only at most twice, checking the repository branch head and -unchanged completed-failure run attempt both before and after the 180-second delay. -Retry code comes from the trusted default branch. PR automatic retries and -cross-run success reuse are deferred until execution evidence can be verified -independently; manual workflow reruns remain available. - -The parent-metadata Maven regression runs once in the selected shared fixture -for Java consumer validation, using a temporary Maven repository. Documentation -and Go-only PRs keep the lightweight planner; the manual fixture workflow also -runs this regression. +Dockerfile-only changes run the image check without building Server fixtures. Hubble's Dockerfile builds Client and Loader before Hubble, so Loader packaging changes also select Hubble. Packaged README changes select images; the existing nonempty Hubble README content contract still requires fresh validation if the file is missing, empty or has an invalid mode. Every selected module and image runs in the current workflow; successful results from another run are not reused. + +Each image job builds the checkout's actual Dockerfile, records its image ID and starts a local run-scoped tag with pulling disabled. The container's image ID must match the build. Loader directly executes `./bin/hugegraph-loader.sh --help`, including its executable permission and shebang; its default idle container alone cannot pass. Hubble must remain running and return a valid `/about` JSON response with the application name and version. Its public root page must contain the React root and serve nonempty local JavaScript bundles and referenced stylesheets with the correct content types, rejecting the HTML fallback for missing assets. These checks validate image build, packaging, startup and public entry points; Loader data ingestion and Hubble browser workflows remain covered by module tests. They do not publish an image. + +Selected Client, Loader, Tools, Spark and Hubble tests compile and run on Java 17, including Loader's separate HDFS jobs. Core module tests cover the locked candidate and released Server 1.7 where configured; these runtime checks supplement affected-module selection without adding unrelated modules. Their results remain advisory. + +The historical Server 1.7 fixture uses the official ASF 1.7.0 binary archive with a pinned official SHA-512 and runs on its own Java 11 JVM. Downloads fail without a source-build fallback. The locked ASF master Server uses Java 17 and TinkerPop 3.8.1. CI locks the complete immutable source commit (shown as `e62c96` here), so later master updates do not change the build inputs. The SDK action, CI fixtures, image defaults and packaging verifier use the same locked source commit. + +The official release archive is downloaded once and cached; Hubble's candidate fixture is built once, with independent services per job. Reuse verifies source, commit, JDK, build inputs and archive checksum. Client, Loader and Tools candidate tests start the archive already produced by their SDK bootstrap. SDK generation remains job-local until distinct published SDK coordinates are available. The bootstrap flattens CI-friendly POM versions before installation, including the root parent, so dependency and license metadata readers consume concrete coordinates without changing the locked source POMs. The service JVM is scoped to startup; Toolchain compilation and tests remain on Java 17. + +The shared Java initialization action requires Maven 3.9 or newer and supplies explicit settings through `MAVEN_ARGS`, preserving existing arguments and user settings. Dependencies are resolved from Central before ASF Stage, with Stage snapshots disabled; build plugins use the standard public repositories. Maven cache keys include repository settings and SDK preparation inputs. Only third-party dependencies seed the isolated SDK repository; cached HugeGraph artifacts are discarded, and Hubble reuses the shared Maven cache without a second overlapping cache. + +Loader validates its existing profiles against both configured Server baselines. The Ubuntu HDFS jobs use the ASF Hadoop 3.3.6 image pinned by digest, with separate NameNode and DataNode containers on the host network. Tests run on Java 17 and use `localhost:8020`. Startup checks live DataNode registration and a real block write/read before testing; containers are removed afterward. Image pulling and readiness are bounded. Cold runners still need to pull the image. + +The immutable Server packages, `ci-plan` and `ci-test-results` diagnostic artifacts are retained for seven days from their upload. Partial reruns need the original plan and fixture within that window. After an artifact expires, rerun the whole workflow to rebuild its plan and fixtures; extending only the result report cannot restore an expired package. Test reports and coverage artifacts keep their existing retention settings. + +Documentation paths use an explicit allowlist. A PR containing only plain prose and static documentation assets has no module tests. Selection uses the cumulative PR diff, so a documentation update to a PR that also changes source runs its selected consumers again. Source, type definitions, tests and CI configuration are never treated as docs. The gate checks the current workflow's actual selected job results and fixture producers. Its result report records the run, attempt and input metadata for diagnostics; it is not a reusable success receipt. The planner determines affected consumers before resolving external Server inputs. Plans that require Server fixtures record the historical release and locked candidate source; Go-only module tests consume only the release fixture. Image-only and plain documentation plans need neither baseline. Unverifiable selection conservatively resolves both. A failed selected baseline lookup fails planning; without a published plan, recovery requires a full workflow rerun. + +PR planning records the event's head, base and source before querying live metadata. A known mismatch with the checkout merge, current head or source fails planning. The report records the actual tested base and merge; advancing master alone does not invalidate a completed result, consistent with non-strict branch protection. Unknown selection falls back to full validation. No cross-run success is reused. + +CodeQL starts independently on affected PR and push inputs, with its original analysis identity and weekly full scan. Java, frontend and Python changes select their supported scan language; shared/unknown build inputs keep all three languages. Go-only and image-only changes do not start unrelated source scans. Manual and weekly runs scan all three languages. Plain prose-only PRs skip compilation, fixtures, images and scanning; native workflows retain lightweight selection jobs. Packaged README files and shared build/configuration inputs are not plain prose. The `CI_OPTIMIZED_REQUIRED` variable is no longer needed. + +The dependency inventory audit includes root `.mvn/**`, POMs, dependency manifests, assembly/license inputs and staged Maven settings. It runs from the repository root with the same root Maven configuration, separately from the required license check. + +For rollout, `.asf.yaml` requests only `check-license-header`. Confirm actual branch protection has reconciled before relying on advisory cancellation. The existing module summary remains available during migration. `Analyze (java)` runs for Java or shared/unknown build inputs, and for manual and weekly full scans; it is not emitted for every PR. Old required names must not remain permanently Expected. + +A new PR head cancels older first attempts. Reruns use separate concurrency groups, so retrying an old commit cannot cancel the current head. Automatic retries run failed **push** license jobs only at most twice, checking the repository branch head and unchanged completed-failure run attempt both before and after the 180-second delay. Retry code comes from the trusted default branch. PR automatic retries and cross-run success reuse are deferred until execution evidence can be verified independently; manual workflow reruns remain available. + +The parent-metadata Maven regression runs once in the selected shared fixture for Java consumer validation, using a temporary Maven repository. Documentation and Go-only PRs keep the lightweight planner; the manual fixture workflow also runs this regression. Validate policy and retry behavior locally: @@ -156,3 +68,5 @@ Validate policy and retry behavior locally: python3 -m unittest discover -s .github/scripts -p 'test_*.py' actionlint ``` + +Upstream compatibility runs daily and on demand against the latest ASF master, resolved once to an immutable commit for each run. It reuses the SDK build, core compilation, Client HTTP tests, Loader and Hubble backend unit tests, and Tools functional tests; Hadoop, Spark/Flink engine matrices and frontend browser tests remain in module CI. This independent check does not change the locked Module validation baseline. diff --git a/docs/java17-migration.md b/docs/java17-migration.md index 497d509c6..cb0353625 100644 --- a/docs/java17-migration.md +++ b/docs/java17-migration.md @@ -2,9 +2,11 @@ Toolchain 1.8 requires Java 17 for the Java Client, ordinary Loader, Tools and Hubble. Select Java 17 for applications and for every Spark driver and executor using the new Client. An older server can keep its own supported JVM. +Toolchain builds, tests and applications use Java 17. The released Server 1.7 compatibility fixture retains its supported Java 11 in a separate process; its JVM must not select the Toolchain build or test JVM. Building the candidate Server SDK requires JDK 17, matching the server reactor's `[17,18)` Enforcer range. + ## Build the locked candidate -The Java 17 server candidate retains the Maven version `1.7.0`. Its Common, PD, gRPC and Store artifacts must come from the same source commit; Maven Central artifacts with that version are not interchangeable with the candidate. The locked source is `apache/hugegraph`, commit `d9abcd4317fb36128e7e4d209139ec7f3a28cdfc` from the upstream master branch after the Java 17 and TinkerPop 3.8 migration. +The ASF Server master uses Java 17 and TinkerPop 3.8.1 while retaining the Maven version `1.7.0`. Its Common, PD, gRPC and Store artifacts must come from the same source commit; Maven Central artifacts with that version are not interchangeable with the candidate. CI uses the verified `apache/hugegraph` master baseline (shown as `e62c96` here), storing and fetching its complete immutable commit identity. Update the SDK action, verifier, Docker defaults and CI baseline together when advancing that commit; moving master alone must not silently change packaged SDK provenance. Use an explicit JDK directory and check both Java and Maven. On systems where Java 17 is not registered, a system JDK selector can return another installed version. Checking the generated class version alone does not identify the JVM that built or tested it. @@ -20,8 +22,11 @@ toolchain_root="$PWD" candidate_dir=$(mktemp -d /tmp/hugegraph-java17.XXXXXX) mkdir "$candidate_dir/m2" export SERVER_REPOSITORY=apache/hugegraph -export SERVER_FETCH_REF=d9abcd4317fb36128e7e4d209139ec7f3a28cdfc -server_commit=d9abcd4317fb36128e7e4d209139ec7f3a28cdfc +server_commit=$(python3 - <<'PY_COMMIT' +import runpy +print(runpy.run_path(".github/scripts/verify_candidate_image_sdk.py")["COMMIT"]) +PY_COMMIT +) bash "$toolchain_root/hugegraph-client/assembly/travis/install-candidate-sdk.sh" \ "$server_commit" "$candidate_dir/server" "$candidate_dir/m2" @@ -29,7 +34,7 @@ mvn -Dmaven.repo.local="$candidate_dir/m2" clean install \ -DskipTests -Dmaven.javadoc.skip=true -ntp ``` -The SDK helper installs the locked Server distribution and its Maven reactor dependencies without starting a server. This includes every required SDK module and parent POM, but excludes unrelated Cluster tests and PD/Store service distributions. It requires a new external source directory and a dedicated Maven repository with no existing HugeGraph artifacts. It records source and artifact provenance in `candidate-sdk-manifest.json` inside that repository. Reuse the same explicit `-Dmaven.repo.local` option for subsequent Toolchain validation. The manifest's source and JDK paths describe that build; use your own directories when rebuilding. The package command skips test execution; run the module test suites separately. These source-built packages are candidates, not an ASF release. +The SDK helper installs the locked Server distribution, PD client, Store client and Struct modules together with their Maven dependency closure, without starting a server. Unrelated Server modules are excluded. It requires a new external source directory and a dedicated Maven repository with no existing HugeGraph artifacts. It records source and artifact provenance in `candidate-sdk-manifest.json` inside that repository. Reuse the same explicit `-Dmaven.repo.local` option for subsequent Toolchain validation. The manifest's source and JDK paths describe that build; use your own directories when rebuilding. The package command skips test execution; run the module test suites separately. These source-built packages are candidates, not an ASF release. Distribution packaging also requires Python 3.9 or newer. Before each module archive is written, the existing SDK verifier checks the locked source, required POM/JAR hashes and local installation origins, then checks the distribution's manifest and actual SDK libraries. The combined archive rechecks all three module directories before moving them. Source checks lock Server SDK artifacts; Toolchain artifacts such as the Client, Loader and Hubble backend are outputs of this reactor and may change during `install`. Older manifests can retain their metadata, but those outputs are excluded from SDK source hashes. The original manifest is still retained unchanged and compared with each packaged copy. A missing manifest, a Central artifact with the same version, or a stale/changed packaged SDK library fails packaging. Ordinary `compile` and `test` do not require the candidate manifest; they may validate published dependencies without establishing candidate package identity. diff --git a/hugegraph-client/README.md b/hugegraph-client/README.md index d2360c1a2..004f112c2 100644 --- a/hugegraph-client/README.md +++ b/hugegraph-client/README.md @@ -5,7 +5,7 @@ [![codecov](https://codecov.io/gh/hugegraph/hugegraph-client/branch/master/graph/badge.svg)](https://codecov.io/gh/hugegraph/hugegraph-client) [![Maven Central](https://img.shields.io/maven-central/v/org.apache.hugegraph/hugegraph-client)](https://mvnrepository.com/artifact/org.apache.hugegraph/hugegraph-client) -hugegraph-client is a Java-written client of [HugeGraph](https://github.com/hugegraph/hugegraph), providing operations of graph, schema, gremlin, variables and traversals etc. All these operations are interpreted and translated into RESTful requests to HugeGraph Server. Besides, hugegraph-client also checks arguments, serializes and deserializes structures and encapsulates server exceptions. +hugegraph-client is a Java-written client of [HugeGraph](https://github.com/apache/hugegraph), providing operations of graph, schema, gremlin, variables and traversals etc. All these operations are interpreted and translated into RESTful requests to HugeGraph Server. Besides, hugegraph-client also checks arguments, serializes and deserializes structures and encapsulates server exceptions. ## Features diff --git a/hugegraph-client/assembly/travis/checkout-server.sh b/hugegraph-client/assembly/travis/checkout-server.sh index 53b543f0f..827ac155f 100644 --- a/hugegraph-client/assembly/travis/checkout-server.sh +++ b/hugegraph-client/assembly/travis/checkout-server.sh @@ -41,7 +41,7 @@ git -C "$CHECKOUT_DIR" fetch --depth 1 -- origin "$SERVER_FETCH_REF" git -C "$CHECKOUT_DIR" checkout --detach FETCH_HEAD ACTUAL_COMMIT=$(git -C "$CHECKOUT_DIR" rev-parse HEAD) if [[ "$ACTUAL_COMMIT" != "$(printf '%s' "$COMMIT_ID" | tr '[:upper:]' '[:lower:]')" ]]; then - echo "Server checkout mismatch: expected $COMMIT_ID, got $ACTUAL_COMMIT" >&2 + echo "Server checkout mismatch: expected ${COMMIT_ID:0:6}, got ${ACTUAL_COMMIT:0:6}" >&2 exit 1 fi -printf 'Server source: %s@%s\n' "$SERVER_REPOSITORY" "$ACTUAL_COMMIT" +printf 'Server source: %s@%s\n' "$SERVER_REPOSITORY" "${ACTUAL_COMMIT:0:6}" diff --git a/hugegraph-client/assembly/travis/install-candidate-sdk.sh b/hugegraph-client/assembly/travis/install-candidate-sdk.sh index 55c6f3f96..4bf788328 100755 --- a/hugegraph-client/assembly/travis/install-candidate-sdk.sh +++ b/hugegraph-client/assembly/travis/install-candidate-sdk.sh @@ -23,13 +23,13 @@ if [[ $# -ne 3 || "$2" != /* || -e "$2" || "$3" != /* || ! -d "$3" || -e "$3/org fi if [[ -z "${JAVA_HOME:-}" || ! -x "$JAVA_HOME/bin/java" ]]; then - echo "Set JAVA_HOME to an installed JDK 17 or newer before building the candidate SDK" >&2 + echo "Set JAVA_HOME to an installed JDK 17 before building the candidate SDK" >&2 exit 1 fi JAVA_VERSION=$("$JAVA_HOME/bin/java" -XshowSettings:properties -version 2>&1 | awk -F ' = ' '/java.specification.version =/ {print $2}') -if [[ ! "$JAVA_VERSION" =~ ^[0-9]+$ || "$JAVA_VERSION" -lt 17 ]]; then - echo "Candidate SDK requires JDK 17 or newer, found: $JAVA_VERSION" >&2 +if [[ ! "$JAVA_VERSION" =~ ^[0-9]+$ || "$JAVA_VERSION" -ne 17 ]]; then + echo "Candidate SDK requires JDK 17, found: $JAVA_VERSION" >&2 exit 1 fi export PATH="$JAVA_HOME/bin:$PATH" @@ -41,16 +41,22 @@ SERVER_FETCH_REF=${SERVER_FETCH_REF:-$1} export SERVER_REPOSITORY SERVER_FETCH_REF CANDIDATE_REPO=$3 bash "$SCRIPT_DIR/checkout-server.sh" "$1" "$SOURCE_DIR" -# Install the Server distribution's dependency reactor so all SDK modules share one source. +# Install the SDK and Server distribution dependency closure from one source. +# Server cluster tests pull an unrelated released Toolchain/SDK back into this build. # The fixture installers retain package-only semantics and use another repository. -# Flatten the root too: license metadata rebuilds installed parent models, whose -# raw ${revision} otherwise becomes an invalid artifact request outside the reactor. +SDK_MODULES=hugegraph-server/hugegraph-dist,hugegraph-pd/hg-pd-client +SDK_MODULES+=,hugegraph-store/hg-store-client,hugegraph-struct +# Flatten the root too: its installed ${revision} otherwise breaks dependency +# model reconstruction while remote-resources generates license metadata. (cd "$SOURCE_DIR" && mvn "-Dmaven.repo.local=$CANDIDATE_REPO" \ - org.codehaus.mojo:flatten-maven-plugin:1.2.7:flatten install \ - -pl hugegraph-server/hugegraph-dist -am \ + -pl "$SDK_MODULES" -am \ + org.codehaus.mojo:flatten-maven-plugin:1.3.0:flatten install \ -Dflatten.mode=resolveCiFriendliesOnly -DupdatePomFile=true \ -DskipTests -Dmaven.javadoc.skip=true -ntp) ACTUAL_COMMIT=$(git -C "$SOURCE_DIR" rev-parse HEAD) +if [[ "$SERVER_FETCH_REF" =~ ^[0-9a-fA-F]{40}$ ]]; then + export SERVER_FETCH_REF=$ACTUAL_COMMIT +fi python3 - "$SOURCE_DIR" "$CANDIDATE_REPO" "${SERVER_REPOSITORY:-apache/hugegraph}" \ "$ACTUAL_COMMIT" "$JAVA_HOME" "$JAVA_VERSION" <<'PY_MANIFEST' import hashlib diff --git a/hugegraph-client/assembly/travis/install-server.test.cjs b/hugegraph-client/assembly/travis/install-server.test.cjs index 8fbd54baa..7c3fad105 100644 --- a/hugegraph-client/assembly/travis/install-server.test.cjs +++ b/hugegraph-client/assembly/travis/install-server.test.cjs @@ -404,8 +404,9 @@ test('candidate SDK installs the verified reactor into an explicit repository an const result = run('checkout'); assert.equal(result.status, 0, result.stderr); assert.deepEqual(readFileSync(env.BUILD_ARGS, 'utf8').trim().split('\n'), - [`-Dmaven.repo.local=${repo}`, 'org.codehaus.mojo:flatten-maven-plugin:1.2.7:flatten', 'install', - '-pl', 'hugegraph-server/hugegraph-dist', '-am', + [`-Dmaven.repo.local=${repo}`, '-pl', + 'hugegraph-server/hugegraph-dist,hugegraph-pd/hg-pd-client,hugegraph-store/hg-store-client,hugegraph-struct', + '-am', 'org.codehaus.mojo:flatten-maven-plugin:1.3.0:flatten', 'install', '-Dflatten.mode=resolveCiFriendliesOnly', '-DupdatePomFile=true', '-DskipTests', '-Dmaven.javadoc.skip=true', '-ntp']); const manifest = JSON.parse(readFileSync(join(repo, 'candidate-sdk-manifest.json'), 'utf8')); @@ -435,10 +436,18 @@ test('candidate SDK rejects a moved source before Maven or manifest publication' assert.equal(existsSync(join(repo, 'candidate-sdk-manifest.json')), false); }); -test('candidate SDK refuses missing or legacy JVMs and existing HugeGraph artifact repositories', t => { +test('candidate SDK refuses missing JVMs, unsupported build JVMs and existing artifact repositories', t => { const { root, repo, env, run } = candidateSdkFixture(t); assert.notEqual(run('missing-jvm', undefined, { JAVA_HOME: '' }).status, 0); assert.notEqual(run('legacy-jvm', undefined, { SDK_JAVA_VERSION: '11' }).status, 0); + for (const version of ['18', '21']) { + const result = run(`unsupported-jvm-${version}`, undefined, { SDK_JAVA_VERSION: version }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /requires JDK 17/); + assert.equal(existsSync(join(root, `unsupported-jvm-${version}`)), false); + } + assert.equal(existsSync(env.BUILD_ARGS), false); + assert.equal(existsSync(join(repo, 'candidate-sdk-manifest.json')), false); mkdirSync(join(repo, 'org/apache/hugegraph'), { recursive: true }); const result = run('existing-repo'); assert.notEqual(result.status, 0); diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/util/IdUtil.java b/hugegraph-client/src/main/java/org/apache/hugegraph/util/IdUtil.java index 7934dfe6c..899c39ced 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/util/IdUtil.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/util/IdUtil.java @@ -20,7 +20,7 @@ import org.apache.commons.lang3.StringUtils; /** - * Copied from HugeGraph(...) + * Copied from HugeGraph(...) */ public final class IdUtil { diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/util/SplicingIdGenerator.java b/hugegraph-client/src/main/java/org/apache/hugegraph/util/SplicingIdGenerator.java index 6acb6493e..9bbf36882 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/util/SplicingIdGenerator.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/util/SplicingIdGenerator.java @@ -21,7 +21,7 @@ import java.util.List; /** - * Copied from HugeGraph(...) + * Copied from HugeGraph(...) */ public class SplicingIdGenerator { diff --git a/hugegraph-hubble/Dockerfile b/hugegraph-hubble/Dockerfile index c498e5677..a57653ef0 100644 --- a/hugegraph-hubble/Dockerfile +++ b/hugegraph-hubble/Dockerfile @@ -39,9 +39,8 @@ COPY .github/scripts/verify_candidate_image_sdk.py /pkg/.github/scripts/ RUN mkdir -p /opt/candidate-m2 \ && if [ ! -f /opt/candidate-m2/candidate-sdk-manifest.json ]; then \ SERVER_REPOSITORY=apache/hugegraph \ - SERVER_FETCH_REF=d9abcd4317fb36128e7e4d209139ec7f3a28cdfc \ bash /pkg/hugegraph-client/assembly/travis/install-candidate-sdk.sh \ - d9abcd4317fb36128e7e4d209139ec7f3a28cdfc /opt/candidate-source /opt/candidate-m2; \ + e62c961e00221569d4f955abbadf60faee45b283 /opt/candidate-source /opt/candidate-m2; \ fi \ && python3 /pkg/.github/scripts/verify_candidate_image_sdk.py /opt/candidate-m2 diff --git a/hugegraph-hubble/README.md b/hugegraph-hubble/README.md index 5228f618d..c7cedf264 100644 --- a/hugegraph-hubble/README.md +++ b/hugegraph-hubble/README.md @@ -195,15 +195,14 @@ Download the toolchain source code. git clone https://github.com/apache/hugegraph-toolchain.git ``` -Compile `hubble`. It depends on the loader and client, so you need to build these dependencies in advance during the compilation process (you can skip this step later). +For the Java 17 candidate, prepare the locked SDK using the [bootstrap instructions](../docs/java17-migration.md#build-the-locked-candidate). Build Client and Loader first, then Hubble with the same isolated Maven repository: ```bash cd hugegraph-toolchain -sudo pip install -r hugegraph-hubble/hubble-dist/assembly/travis/requirements.txt -mvn install -pl hugegraph-client,hugegraph-loader -am -Dmaven.javadoc.skip=true -DskipTests -ntp -cd hugegraph-hubble -mvn -e compile package -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -ntp -cd apache-hugegraph-hubble-* +python3 -m pip install -r hugegraph-hubble/hubble-dist/assembly/travis/requirements.txt +mvn -Dmaven.repo.local="$candidate_dir/m2" install -pl hugegraph-client,hugegraph-loader -am -Dmaven.javadoc.skip=true -DskipTests -ntp +(cd hugegraph-hubble && mvn -Dmaven.repo.local="$candidate_dir/m2" package -Dmaven.javadoc.skip=true -DskipTests -ntp) +cd hugegraph-hubble/apache-hugegraph-hubble-* ``` Run `hubble` @@ -214,6 +213,12 @@ bin/start-hubble.sh -d ### 3. User docker image (Convenient for Test/Dev) +To build the current checkout, run from the Toolchain repository root: + +```bash +docker build -f hugegraph-hubble/Dockerfile -t hugegraph/hugegraph-hubble:latest . +``` + We can use `docker run -itd --name=hubble -p 8088:8088 hugegraph/hubble` to quickly start [hubble](https://hub.docker.com/r/hugegraph/hubble). An you can visit [hubble deploy doc](https://hugegraph.apache.org/docs/quickstart/hugegraph-hubble/#2-deploy) for more details. Then we should follow the [hubble workflow doc](https://hugegraph.apache.org/docs/quickstart/hugegraph-hubble/#3platform-workflow) to create the graph. diff --git a/hugegraph-hubble/hubble-be/pom.xml b/hugegraph-hubble/hubble-be/pom.xml index 6d4d83e05..646a20476 100644 --- a/hugegraph-hubble/hubble-be/pom.xml +++ b/hugegraph-hubble/hubble-be/pom.xml @@ -264,12 +264,16 @@ com.sun.xml.bind jaxb-impl - + org.apache.hugegraph hg-store-common + + + org.apdplat + word + diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server1/install-hugegraph.sh b/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server1/install-hugegraph.sh index e22479c22..06c7a4c36 100755 --- a/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server1/install-hugegraph.sh +++ b/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server1/install-hugegraph.sh @@ -27,9 +27,24 @@ ARCHIVES=(apache-hugegraph-*.tar.gz) TAR=${ARCHIVES[0]} tar -zxvf "$TAR" -C "${SERVER_PARENT_DIR}" >/dev/null 2>&1 -HUGEGRAPH_NAME=${TAR%%.tar*} -SERVER_DIR="${SERVER_PARENT_DIR}"/$HUGEGRAPH_NAME -echo $SERVER_DIR +# Official releases wrap Server alongside PD/Store; candidate archives do not. +# Remove macOS sidecars before graph configuration discovery, as Client CI does. +find "$SERVER_PARENT_DIR" -type f -name '._*' -delete +SERVER_DIRS=() +for dir in "$SERVER_PARENT_DIR"/apache-hugegraph-*/ "$SERVER_PARENT_DIR"/apache-hugegraph-*/*/; do + if [[ -f "$dir/conf/graphs/hugegraph.properties" && + -f "$dir/conf/rest-server.properties" && + -f "$dir/conf/gremlin-server.yaml" && + -x "$dir/bin/init-store.sh" && -x "$dir/bin/start-hugegraph.sh" ]]; then + SERVER_DIRS+=("$dir") + fi +done +if [[ ${#SERVER_DIRS[@]} -ne 1 ]]; then + printf 'Expected exactly one runnable server distribution, found %s\n' "${#SERVER_DIRS[@]}" >&2 + exit 1 +fi +SERVER_DIR=${SERVER_DIRS[0]} +echo "$SERVER_DIR" python3 "${SERVER_CONFIG_DIR}/../configure_gremlin_fixture.py" \ "${SERVER_DIR}" "${SERVER_CONFIG_DIR}/gremlin-server.yaml" \ diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server2/install-hugegraph.sh b/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server2/install-hugegraph.sh index 224240b74..f07adf740 100755 --- a/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server2/install-hugegraph.sh +++ b/hugegraph-hubble/hubble-dist/assembly/travis/hugegraph-server2/install-hugegraph.sh @@ -27,9 +27,24 @@ ARCHIVES=(apache-hugegraph-*.tar.gz) TAR=${ARCHIVES[0]} tar -zxvf "$TAR" -C "${SERVER_PARENT_DIR}" >/dev/null 2>&1 -HUGEGRAPH_NAME=${TAR%%.tar*} -SERVER_DIR="${SERVER_PARENT_DIR}"/$HUGEGRAPH_NAME -echo $SERVER_DIR +# Official releases wrap Server alongside PD/Store; candidate archives do not. +# Remove macOS sidecars before graph configuration discovery, as Client CI does. +find "$SERVER_PARENT_DIR" -type f -name '._*' -delete +SERVER_DIRS=() +for dir in "$SERVER_PARENT_DIR"/apache-hugegraph-*/ "$SERVER_PARENT_DIR"/apache-hugegraph-*/*/; do + if [[ -f "$dir/conf/graphs/hugegraph.properties" && + -f "$dir/conf/rest-server.properties" && + -f "$dir/conf/gremlin-server.yaml" && + -x "$dir/bin/init-store.sh" && -x "$dir/bin/start-hugegraph.sh" ]]; then + SERVER_DIRS+=("$dir") + fi +done +if [[ ${#SERVER_DIRS[@]} -ne 1 ]]; then + printf 'Expected exactly one runnable server distribution, found %s\n' "${#SERVER_DIRS[@]}" >&2 + exit 1 +fi +SERVER_DIR=${SERVER_DIRS[0]} +echo "$SERVER_DIR" python3 "${SERVER_CONFIG_DIR}/../configure_gremlin_fixture.py" \ "${SERVER_DIR}" "${SERVER_CONFIG_DIR}/gremlin-server.yaml" \ diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/run_live_hubble_smoke.py b/hugegraph-hubble/hubble-dist/assembly/travis/run_live_hubble_smoke.py index 0f6031a96..13dc3f1e8 100755 --- a/hugegraph-hubble/hubble-dist/assembly/travis/run_live_hubble_smoke.py +++ b/hugegraph-hubble/hubble-dist/assembly/travis/run_live_hubble_smoke.py @@ -122,12 +122,28 @@ def assert_safe_tar_member(member, work_dir): raise RuntimeError(f"Unsafe tar link outside work dir: {member.name}") -def extract_tarball(tarball, work_dir): - with tarfile.open(tarball, "r:gz") as archive: +def require_safe_archive_extraction(): + if not callable(getattr(tarfile, "data_filter", None)): + raise RuntimeError( + "Safe archive extraction requires Python with tarfile.data_filter; " + "upgrade to the latest security patch of Python 3.10 or newer " + "(extraction filters first appeared in 3.9.17, 3.10.12 and 3.11.4). " + "Unfiltered extraction is not supported." + ) + + +def extract_archive(tarball, work_dir): + require_safe_archive_extraction() + with tarfile.open(tarball) as archive: members = archive.getmembers() for member in members: assert_safe_tar_member(member, work_dir) - archive.extractall(work_dir, members) + # Recheck each member against links already written during extraction. + archive.extractall(work_dir, members, filter="data") + + +def extract_tarball(tarball, work_dir): + extract_archive(tarball, work_dir) homes = [path for path in work_dir.iterdir() if path.is_dir() and path.name.startswith("apache-hugegraph-hubble-")] if not homes: @@ -749,6 +765,12 @@ def main(): "pa")) args = parser.parse_args() + if not args.skip_start: + try: + require_safe_archive_extraction() + except RuntimeError as exc: + parser.error(str(exc)) + hubble_url = args.hubble_url.rstrip("/") server_url = args.server_url.rstrip("/") work_dir = args.work_dir or Path(tempfile.mkdtemp(prefix="hubble-694-live-")) diff --git a/hugegraph-hubble/hubble-dist/assembly/travis/test_run_live_hubble_smoke.py b/hugegraph-hubble/hubble-dist/assembly/travis/test_run_live_hubble_smoke.py index 78eb8b957..a3612c4ff 100644 --- a/hugegraph-hubble/hubble-dist/assembly/travis/test_run_live_hubble_smoke.py +++ b/hugegraph-hubble/hubble-dist/assembly/travis/test_run_live_hubble_smoke.py @@ -17,6 +17,8 @@ # import importlib.util +import io +import tarfile import tempfile import unittest from pathlib import Path @@ -29,6 +31,78 @@ SPEC.loader.exec_module(SMOKE) +class ArchiveExtractionTest(unittest.TestCase): + + def test_missing_data_filter_fails_before_opening_archive(self): + with mock.patch.object(tarfile, "data_filter", None, create=True), \ + mock.patch.object(tarfile, "open") as open_archive: + with self.assertRaisesRegex(RuntimeError, "tarfile.data_filter"): + SMOKE.extract_archive(Path("hubble.tar.gz"), Path("runtime")) + open_archive.assert_not_called() + + def test_cli_rejects_unsupported_python_before_runtime_setup(self): + with mock.patch.object(tarfile, "data_filter", None, create=True), \ + mock.patch("sys.argv", [str(SCRIPT), "hubble.tar.gz"]), \ + mock.patch.object(SMOKE, "is_healthy") as is_healthy, \ + mock.patch.object(tempfile, "mkdtemp") as create_work_dir, \ + mock.patch("sys.stderr", new_callable=io.StringIO) as stderr: + with self.assertRaises(SystemExit) as error: + SMOKE.main() + self.assertEqual(2, error.exception.code) + self.assertIn("upgrade to the latest security patch", stderr.getvalue()) + is_healthy.assert_not_called() + create_work_dir.assert_not_called() + + def test_sequential_symlink_escape_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + work = root / "runtime" + work.mkdir() + tarball = root / "escape.tar.gz" + with tarfile.open(tarball, "w:gz") as archive: + for name, target in (("pivot", "."), ("pivot/escape", "..")): + member = tarfile.TarInfo(name) + member.type = tarfile.SYMTYPE + member.linkname = target + archive.addfile(member) + member = tarfile.TarInfo("pivot/escape/escaped") + member.size = 7 + archive.addfile(member, io.BytesIO(b"outside")) + # Each entry passes the old preflight before any links exist. + with tarfile.open(tarball) as archive: + for member in archive.getmembers(): + SMOKE.assert_safe_tar_member(member, work) + + with self.assertRaises(tarfile.FilterError): + SMOKE.extract_archive(tarball, work) + self.assertFalse((root / "escaped").exists()) + + def test_internal_symbolic_and_hard_links_are_preserved(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + work = root / "runtime" + work.mkdir() + tarball = root / "hubble.tar.gz" + home_name = "apache-hugegraph-hubble-fixture" + with tarfile.open(tarball, "w:gz") as archive: + member = tarfile.TarInfo(f"{home_name}/lib/payload") + member.size = 7 + archive.addfile(member, io.BytesIO(b"fixture")) + for name, target, kind in ( + ("current", "lib", tarfile.SYMTYPE), + ("copy", f"{home_name}/lib/payload", tarfile.LNKTYPE)): + member = tarfile.TarInfo(f"{home_name}/{name}") + member.type = kind + member.linkname = target + archive.addfile(member) + + home = SMOKE.extract_tarball(tarball, work) + self.assertEqual(work / home_name, home) + self.assertTrue((home / "current").is_symlink()) + self.assertEqual(b"fixture", (home / "current/payload").read_bytes()) + self.assertEqual(b"fixture", (home / "copy").read_bytes()) + + class HubbleReadinessTest(unittest.TestCase): def test_configure_endpoint_uses_registered_server_options(self): diff --git a/hugegraph-loader/Dockerfile b/hugegraph-loader/Dockerfile index e693a6b79..1985afe64 100644 --- a/hugegraph-loader/Dockerfile +++ b/hugegraph-loader/Dockerfile @@ -35,9 +35,8 @@ COPY .github/scripts/verify_candidate_image_sdk.py /pkg/.github/scripts/ RUN mkdir -p /opt/candidate-m2 \ && if [ ! -f /opt/candidate-m2/candidate-sdk-manifest.json ]; then \ SERVER_REPOSITORY=apache/hugegraph \ - SERVER_FETCH_REF=d9abcd4317fb36128e7e4d209139ec7f3a28cdfc \ bash /pkg/hugegraph-client/assembly/travis/install-candidate-sdk.sh \ - d9abcd4317fb36128e7e4d209139ec7f3a28cdfc /opt/candidate-source /opt/candidate-m2; \ + e62c961e00221569d4f955abbadf60faee45b283 /opt/candidate-source /opt/candidate-m2; \ fi \ && python3 /pkg/.github/scripts/verify_candidate_image_sdk.py /opt/candidate-m2 diff --git a/hugegraph-loader/README.md b/hugegraph-loader/README.md index 83abc79b4..9b660f792 100644 --- a/hugegraph-loader/README.md +++ b/hugegraph-loader/README.md @@ -56,11 +56,10 @@ Install ojdbc8 to the local maven repository, enter the directory where ojdbc8.j mvn install:install-file -Dfile=./ojdbc8.jar -DgroupId=com.oracle -DartifactId=ojdbc8 -Dversion=12.2.0.1 -Dpackaging=jar ``` -Compile and generate tar package: +For the Java 17 candidate, prepare the locked SDK using the [bootstrap instructions](../docs/java17-migration.md#build-the-locked-candidate), then compile from the Toolchain repository root with the same isolated Maven repository: -``` -cd hugegraph-loader -mvn clean package -DskipTests +```bash +mvn -Dmaven.repo.local="$candidate_dir/m2" clean package -pl hugegraph-client,hugegraph-loader -am -DskipTests -ntp ``` ### 2.3 Use docker image (Convenient for Test/Dev) @@ -72,6 +71,12 @@ Use the command `docker run -itd --name loader hugegraph/loader` to start loader If you want to load your data, you can mount the data folder like `-v /path/to/data/file:/loader/file` +To build the current checkout, run from the Toolchain repository root: + +```bash +docker build -f hugegraph-loader/Dockerfile -t hugegraph/hugegraph-loader:latest . +``` + #### 2.3.2 Docker-compose The example `docker-compose.yml` is [here](./docker/example/docker-compose.yml). Use the command `docker-compose up -d` to deploy `loader` with `server` and `hubble`. @@ -121,14 +126,7 @@ docker exec -it loader bin/hugegraph-loader.sh -g hugegraph -f example/file/stru The [loader homepage](https://hugegraph.apache.org/docs/quickstart/hugegraph-loader/) contains more information about it. -The CI server installer `assembly/travis/install-hugegraph-from-source.sh ` -uses `~/hugegraph-cache-` by default. `SERVER_CACHE_DIR` selects the exact cache -directory. A populated cache must contain one server archive and its generated -`server-provenance` file matching the repository, commit, archive name and SHA-256. -Existing custom caches without provenance and mismatched caches are preserved and -rejected; use a fresh directory to build a verified cache, and separate directories for -different server baselines. Loader CI uses a `provenance-v1` cache key so its first run -builds a cache with provenance. The installer does not add SHA subdirectories to `SERVER_CACHE_DIR`. +Server 1.7 compatibility CI uses the pinned official ASF 1.7.0 binary archive. The fixture cache verifies the release version, official source URL, official SHA-512 and cached archive SHA-256 before startup. A failed download does not fall back to rebuilding the release tag. Unpublished candidate SDKs and their Server fixture continue to use the locked ASF source build. ## 5. License diff --git a/hugegraph-loader/assembly/travis/install-hugegraph-from-tar.sh b/hugegraph-loader/assembly/travis/install-hugegraph-from-tar.sh deleted file mode 100755 index 07a4bb335..000000000 --- a/hugegraph-loader/assembly/travis/install-hugegraph-from-tar.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with this -# work for additional information regarding copyright ownership. The ASF -# licenses this file to You under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -set -ev - -if [[ $# -ne 1 ]]; then - echo "Must input an existing commit id of hugegraph server" && exit 1 -fi - -VERSION=$1 -HUGEGRAPH_LINK="https://github.com/hugegraph/hugegraph/releases/download/v${VERSION}/hugegraph-${VERSION}.tar.gz" - -wget "${HUGEGRAPH_LINK}" || exit 1 -tar zxvf hugegraph-${VERSION}.tar.gz - -HTTPS_SERVER_DIR="hugegraph_https" -mkdir ${HTTPS_SERVER_DIR} -cp -r hugegraph-${VERSION}/. ${HTTPS_SERVER_DIR} -cd hugegraph-${VERSION} - -# start HugeGraphServer with http protocol -bin/init-store.sh || exit 1 -bin/start-hugegraph.sh || exit 1 - -cd ../${HTTPS_SERVER_DIR} -REST_SERVER_CONFIG="conf/rest-server.properties" -GREMLIN_SERVER_CONFIG="conf/gremlin-server.yaml" -sed -i "s?http://127.0.0.1:8080?https://127.0.0.1:8443?g" "$REST_SERVER_CONFIG" -sed -i "s/#port: 8182/port: 8282/g" "$GREMLIN_SERVER_CONFIG" -echo "gremlinserver.url=http://127.0.0.1:8282" >> $REST_SERVER_CONFIG - -# start HugeGraphServer with https protocol -bin/init-store.sh -bin/start-hugegraph.sh -cd ../