From 193843ac84ced5d7e022eaf44dbc39afceb7c759 Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 8 Oct 2026 20:12:17 +0800 Subject: [PATCH 1/5] feat(loader): migrate CDC to Flink 2.2 - adapt CDC loading to Sink V2 on Java 17 - preserve same-identity updates and reject identity changes - retain packaged recovery and class-loading validation - include Flink runtime packaging and license inventory --- .github/workflows/flink-cdc-ci.yml | 110 ++ hugegraph-dist/release-docs/LICENSE | 151 ++- hugegraph-dist/release-docs/NOTICE | 507 ++++++++ .../LICENSE-flink-final-third-party.txt | 1154 +++++++++++++++++ .../scripts/dependency/known-dependencies.txt | 67 +- hugegraph-loader/README.md | 2 + .../assembly/descriptor/assembly.xml | 48 + .../static/bin/hugegraph-flinkcdc-loader.sh | 46 +- .../assembly/static/bin/hugegraph-loader.sh | 3 +- .../assembly/travis/flink-cdc/test_verify.py | 154 +++ .../assembly/travis/flink-cdc/verify.py | 475 +++++++ hugegraph-loader/docs/flink-cdc-java17.md | 101 ++ hugegraph-loader/pom.xml | 40 +- .../hugegraph/loader/builder/EdgeBuilder.java | 10 + .../loader/builder/VertexBuilder.java | 11 +- .../flink/HugeGraphDeserialization.java | 33 +- .../loader/flink/HugeGraphFlinkCDCLoader.java | 21 +- .../loader/flink/HugeGraphOutputFormat.java | 244 +++- .../hugegraph/loader/flink/HugeGraphSink.java | 72 + .../loader/flink/HugeGraphSinkFunction.java | 68 - .../flink/HugeGraphDeserializationTest.java | 147 +++ .../HugeGraphIdentityIntegrationTest.java | 315 +++++ .../loader/flink/HugeGraphSinkTest.java | 714 ++++++++++ .../flink/PackagedCdcClassLoadingProbe.java | 107 ++ .../test/unit/FlinkSinkSerializationTest.java | 85 ++ .../loader/test/unit/UnitTestSuite.java | 5 +- .../src/test/scripts/flink-launcher.test.cjs | 87 ++ pom.xml | 4 +- 28 files changed, 4633 insertions(+), 148 deletions(-) create mode 100644 .github/workflows/flink-cdc-ci.yml create mode 100644 hugegraph-dist/release-docs/licenses/LICENSE-flink-final-third-party.txt create mode 100644 hugegraph-loader/assembly/travis/flink-cdc/test_verify.py create mode 100644 hugegraph-loader/assembly/travis/flink-cdc/verify.py create mode 100644 hugegraph-loader/docs/flink-cdc-java17.md create mode 100644 hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphSink.java delete mode 100644 hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphSinkFunction.java create mode 100644 hugegraph-loader/src/test/java/org/apache/hugegraph/loader/flink/HugeGraphDeserializationTest.java create mode 100644 hugegraph-loader/src/test/java/org/apache/hugegraph/loader/flink/HugeGraphIdentityIntegrationTest.java create mode 100644 hugegraph-loader/src/test/java/org/apache/hugegraph/loader/flink/HugeGraphSinkTest.java create mode 100644 hugegraph-loader/src/test/java/org/apache/hugegraph/loader/flink/PackagedCdcClassLoadingProbe.java create mode 100644 hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/FlinkSinkSerializationTest.java create mode 100644 hugegraph-loader/src/test/scripts/flink-launcher.test.cjs diff --git a/.github/workflows/flink-cdc-ci.yml b/.github/workflows/flink-cdc-ci.yml new file mode 100644 index 000000000..244cdf48b --- /dev/null +++ b/.github/workflows/flink-cdc-ci.yml @@ -0,0 +1,110 @@ +name: "flink-cdc-java17-ci" + +on: + workflow_dispatch: + pull_request: + paths: + - hugegraph-loader/** + - hugegraph-client/** + - pom.xml + - .github/workflows/flink-cdc-ci.yml + - .github/actions/get-hugegraph-commit/** + - .github/actions/setup-candidate-sdk/** + push: + branches: [master, 'release-*'] + paths: + - hugegraph-loader/** + - hugegraph-client/** + - pom.xml + - .github/workflows/flink-cdc-ci.yml + - .github/actions/get-hugegraph-commit/** + - .github/actions/setup-candidate-sdk/** + +permissions: + contents: read + +# Keep the newest PR run; preserve independent push, release and manual runs. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + packaged-cdc-recovery: + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + MAVEN_OPTS: -Xmx768m -XX:ActiveProcessorCount=2 + FLINK_VERSION: '2.2.1' + CDC_VERSION: '3.6.0-2.2' + steps: + - uses: actions/checkout@v7 + + - name: Prepare Java 17 + uses: actions/setup-java@v5 + with: + distribution: zulu + java-version: '17' + + - name: Install locked same-source candidate SDK + id: candidate-sdk + uses: ./.github/actions/setup-candidate-sdk + + - name: Validate acceptance tooling + run: | + node --test hugegraph-loader/src/test/scripts/flink-launcher.test.cjs + python3 -m unittest discover -s hugegraph-loader/assembly/travis/flink-cdc -p 'test_*.py' + python3 hugegraph-loader/assembly/travis/flink-cdc/verify.py + + - name: Build actual Loader distribution job + run: | + mvn package -pl hugegraph-client,hugegraph-loader -am -DskipTests -Dmaven.javadoc.skip=true \ + -Dflink.version="$FLINK_VERSION" -Dflinkcdc.version="$CDC_VERSION" -ntp + mvn test checkstyle:check -pl hugegraph-client,hugegraph-loader -am -DforkCount=0 \ + -Dtest=FlinkSinkSerializationTest,HugeGraphSinkTest,HugeGraphDeserializationTest \ + -Dsurefire.failIfNoSpecifiedTests=false \ + -Dflink.version="$FLINK_VERSION" -Dflinkcdc.version="$CDC_VERSION" -ntp + + - name: Obtain external test-only JDBC driver + run: | + mkdir -p "$RUNNER_TEMP/flink-test-drivers" + curl --fail --location --retry 3 \ + https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.28/mysql-connector-java-8.0.28.jar \ + --output "$RUNNER_TEMP/flink-test-drivers/mysql-connector-java-8.0.28.jar" + # The harness verifies its pinned SHA-256 before starting any runtime. + + - name: Link packaged CDC coordinator against the actual Flink runtime + run: | + jobs=(hugegraph-loader/target/*-shaded.jar) + [[ ${#jobs[@]} -eq 1 && -f "${jobs[0]}" ]] + docker run --rm --network none --memory 512m --entrypoint java \ + -v "$PWD/${jobs[0]}:/job.jar:ro" \ + -v "$PWD/hugegraph-loader/target/test-classes:/probe:ro" \ + "flink:${FLINK_VERSION}-java17" -Xmx256m -XX:ActiveProcessorCount=2 -cp /probe \ + org.apache.hugegraph.loader.flink.PackagedCdcClassLoadingProbe /job.jar /opt/flink/lib + + - name: Verify packaged CDC snapshot and recovery + env: + SERVER_COMMIT: ${{ steps.candidate-sdk.outputs.commit_id }} + CANDIDATE_ARCHIVE: ${{ steps.candidate-sdk.outputs.archive }} + CANDIDATE_ARCHIVE_SHA256: ${{ steps.candidate-sdk.outputs.archive-sha256 }} + CANDIDATE_SDK_MANIFEST: ${{ steps.candidate-sdk.outputs.manifest }} + CANDIDATE_MAVEN_REPO: ${{ steps.candidate-sdk.outputs.maven-repo }} + run: | + jobs=(hugegraph-loader/target/*-shaded.jar) + [[ -f "$CANDIDATE_ARCHIVE" && -f "$CANDIDATE_SDK_MANIFEST" ]] + [[ ${#jobs[@]} -eq 1 && -f "${jobs[0]}" ]] + python3 hugegraph-loader/assembly/travis/flink-cdc/verify.py --run \ + --server-archive "$CANDIDATE_ARCHIVE" --job-jar "${jobs[0]}" \ + --server-archive-sha256 "$CANDIDATE_ARCHIVE_SHA256" --sdk-manifest "$CANDIDATE_SDK_MANIFEST" \ + --maven-repo "$CANDIDATE_MAVEN_REPO" \ + --jdbc-driver "$RUNNER_TEMP/flink-test-drivers/mysql-connector-java-8.0.28.jar" \ + --server-sha "$SERVER_COMMIT" --toolchain-sha "$(git rev-parse HEAD)" \ + --work-root "$RUNNER_TEMP" --evidence "$RUNNER_TEMP/flink-cdc-evidence" + + - name: Upload non-sensitive runtime evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: flink-cdc-java17-evidence + path: ${{ runner.temp }}/flink-cdc-evidence + if-no-files-found: warn diff --git a/hugegraph-dist/release-docs/LICENSE b/hugegraph-dist/release-docs/LICENSE index eeccf5fdb..71412b9d6 100644 --- a/hugegraph-dist/release-docs/LICENSE +++ b/hugegraph-dist/release-docs/LICENSE @@ -403,7 +403,6 @@ See licenses/ for text of these licenses. (Apache License, Version 2.0) * opencsv (net.sf.opencsv:opencsv:2.3 - http://opencsv.sf.net) (Apache License, Version 2.0) * snappy-java (org.xerial.snappy:snappy-java:1.1.8.2 - https://github.com/xerial/snappy-java) (Apache License, Version 2.0) * snappy-java (org.xerial.snappy:snappy-java:1.1.8.4 - https://github.com/xerial/snappy-java) - (Apache License, Version 2.0) * Jetty :: Asynchronous HTTP Client (org.eclipse.jetty:jetty-client:9.4.33.v20201020 - https://eclipse.org/jetty/jetty-client) (Apache License, Version 2.0) * Jetty :: Asynchronous HTTP Client (org.eclipse.jetty:jetty-client:9.4.40.v20210413 - https://eclipse.org/jetty/jetty-client) (Apache License, Version 2.0) * Jetty :: Http Utility (org.eclipse.jetty:jetty-http:9.4.40.v20210413 - https://eclipse.org/jetty/jetty-http) (Apache License, Version 2.0) * Jetty :: IO Utility (org.eclipse.jetty:jetty-io:9.4.40.v20210413 - https://eclipse.org/jetty/jetty-io) @@ -1084,3 +1083,153 @@ represented as byte-identical Maven Central 1.7.0 artifacts. Apache-2.0 terms above (Apache-2.0) org.apiguardian:apiguardian-api:1.1.2 licenses/LICENSE-jackson-datatype-jdk8.txt + +======================================================================== +Flink CDC runtime dependency additions +======================================================================== + +The exact runtime union adds the components below. Versions retained by other +modules remain listed separately. Apache-2.0 terms are reproduced above; +other complete grants and bundled attributions are in the referenced files. +Eclipse/Jersey/HK2/JAX-RS components use EPL-2.0, Jetty uses Apache-2.0, +JavaBeans Activation 1.1.1 uses CDDL-1.0, and Reflections uses WTFPL. +Zstd JNI and its bundled native Zstandard library use BSD-2-Clause. +Flink engine and MySQL JDBC driver are supplied by the deployment environment; +the runtime dependency model and Loader shaded artifact exclude those inputs. +The MySQL binlog connector is an Apache-2.0 component, distinct from the JDBC driver. +Required full notices are reproduced in the main distribution NOTICE. +The supplement also retains the actual shaded payload's inherited license +resources for JAXB/Commons, gRPC native code, Groovy, PostgreSQL SCRAM/stringprep, +DataNucleus ASM and the Arrow Apache header template. These preserve baseline +terms and attributions; they are not new CDC dependency identities. + + (Apache-2.0) at.yawk.lz4:lz4-java:1.10.1 + Apache-2.0 terms above + (Apache-2.0) ch.qos.reload4j:reload4j:1.2.19 + licenses/LICENSE-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.txt + (Apache-2.0) ch.qos.reload4j:reload4j:1.2.25 + licenses/LICENSE-listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.txt + (Apache-2.0) com.esri.geometry:esri-geometry-api:2.2.0 + Apache-2.0 terms above + (Apache-2.0) com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.12.3 + Apache-2.0 terms above + (Apache-2.0) com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.12.3 + licenses/LICENSE-jackson-core.txt + (Apache-2.0) com.fasterxml.jackson.module:jackson-module-afterburner:2.21.4 + licenses/LICENSE-flink-final-third-party.txt + (BSD-2-Clause) com.github.luben:zstd-jni:1.5.0-2 + licenses/LICENSE-flink-final-third-party.txt, licenses/LICENSE-hubble-java17-third-party.txt + (BSD-2-Clause) com.github.luben:zstd-jni:1.5.0-4 + licenses/LICENSE-flink-final-third-party.txt, licenses/LICENSE-hubble-java17-third-party.txt + (Apache-2.0) com.zaxxer:HikariCP:4.0.3 + Apache-2.0 terms above + (Apache-2.0) com.zendesk:mysql-binlog-connector-java:0.27.2 + Apache-2.0 terms above + (Apache-2.0) io.debezium:debezium-api:1.9.8.Final + Apache-2.0 terms above + (Apache-2.0) io.debezium:debezium-connector-mysql:1.9.8.Final + Apache-2.0 terms above + (Apache-2.0) io.debezium:debezium-core:1.9.8.Final + Apache-2.0 terms above + (Apache-2.0) io.debezium:debezium-ddl-parser:1.9.8.Final + Apache-2.0 terms above + (Apache-2.0) io.debezium:debezium-embedded:1.9.8.Final + Apache-2.0 terms above + (Apache-2.0) io.swagger.core.v3:swagger-annotations:2.2.8 + licenses/LICENSE-flink-final-third-party.txt + (EPL-2.0) jakarta.ws.rs:jakarta.ws.rs-api:2.1.6 + licenses/LICENSE-hubble-java17-third-party.txt + (CDDL-1.0) javax.activation:activation:1.1.1 + licenses/LICENSE-flink-final-third-party.txt + (EPL-2.0) javax.ws.rs:javax.ws.rs-api:2.1.1 + licenses/LICENSE-hubble-java17-third-party.txt + (MIT) net.sourceforge.argparse4j:argparse4j:0.7.0 + licenses/LICENSE-flink-final-third-party.txt + (BSD-3-Clause) org.antlr:antlr4-runtime:4.8 + licenses/LICENSE-java17-candidate-sdk.txt + (Apache-2.0) org.apache.flink:flink-cdc-common:3.6.0-2.2 + Apache-2.0 terms above + (Apache-2.0) org.apache.flink:flink-cdc-flink2-compat:3.6.0-2.2 + Apache-2.0 terms above + (Apache-2.0) org.apache.flink:flink-cdc-runtime:3.6.0-2.2 + Apache-2.0 terms above + (Apache-2.0) org.apache.flink:flink-connector-debezium:3.6.0-2.2 + Apache-2.0 terms above + (Apache-2.0) org.apache.flink:flink-connector-mysql-cdc:3.6.0-2.2 + Apache-2.0 terms above + (Apache-2.0) org.apache.flink:flink-shaded-guava:31.1-jre-17.0 + Apache-2.0 terms above + (Apache-2.0) org.apache.flink:flink-shaded-guava:33.4.0-jre-20.0 + Apache-2.0 terms above + (Apache-2.0) org.apache.kafka:connect-api:3.2.0 + Apache-2.0 terms above + (Apache-2.0) org.apache.kafka:connect-api:3.9.2 + Apache-2.0 terms above + (Apache-2.0) org.apache.kafka:connect-file:3.2.0 + Apache-2.0 terms above + (Apache-2.0) org.apache.kafka:connect-file:3.9.2 + Apache-2.0 terms above + (Apache-2.0) org.apache.kafka:connect-json:3.2.0 + Apache-2.0 terms above + (Apache-2.0) org.apache.kafka:connect-json:3.9.2 + Apache-2.0 terms above + (Apache-2.0) org.apache.kafka:connect-runtime:3.2.0 + Apache-2.0 terms above + (Apache-2.0) org.apache.kafka:connect-runtime:3.9.2 + Apache-2.0 terms above + (Apache-2.0) org.apache.kafka:connect-transforms:3.2.0 + Apache-2.0 terms above + (Apache-2.0) org.apache.kafka:connect-transforms:3.9.2 + Apache-2.0 terms above + (Apache-2.0) org.apache.kafka:kafka-clients:3.2.0 + Apache-2.0 terms above + (Apache-2.0) org.apache.kafka:kafka-clients:3.9.2 + Apache-2.0 terms above + (Apache-2.0) org.apache.kafka:kafka-tools:3.2.0 + Apache-2.0 terms above + (Apache-2.0) org.apache.maven:maven-artifact:3.8.4 + Apache-2.0 terms above + (Apache-2.0) org.apache.maven:maven-artifact:3.9.6 + Apache-2.0 terms above + (Apache-2.0) org.bitbucket.b_c:jose4j:0.7.9 + Apache-2.0 terms above + (Apache-2.0) org.bitbucket.b_c:jose4j:0.9.6 + Apache-2.0 terms above + (Apache-2.0) org.codehaus.plexus:plexus-utils:3.3.0 + Apache-2.0 terms above, licenses/LICENSE-flink-final-third-party.txt + (Apache-2.0) org.codehaus.plexus:plexus-utils:3.5.1 + Apache-2.0 terms above, licenses/LICENSE-flink-final-third-party.txt + (Apache-2.0) org.eclipse.jetty:jetty-alpn-client:12.0.36 + Apache-2.0 terms above + (Apache-2.0) org.eclipse.jetty:jetty-client:12.0.36 + Apache-2.0 terms above + (Apache-2.0) org.eclipse.jetty:jetty-client:9.4.44.v20210927 + licenses/LICENSE-jetty-client.txt + (Apache-2.0) org.eclipse.jetty:jetty-continuation:9.4.44.v20210927 + licenses/LICENSE-jetty-client.txt + (Apache-2.0) org.eclipse.jetty:jetty-continuation:9.4.57.v20241219 + licenses/LICENSE-jetty-client.txt + (Apache-2.0) org.eclipse.jetty:jetty-servlets:9.4.44.v20210927 + licenses/LICENSE-jetty-client.txt + (Apache-2.0) org.eclipse.jetty:jetty-servlets:9.4.57.v20241219 + licenses/LICENSE-jetty-client.txt + (EPL-2.0) org.glassfish.hk2.external:aopalliance-repackaged:2.6.1 + licenses/LICENSE-hubble-java17-third-party.txt + (EPL-2.0) org.glassfish.hk2.external:jakarta.inject:2.6.1 + licenses/LICENSE-hubble-java17-third-party.txt + (EPL-2.0) org.glassfish.hk2:hk2-api:2.6.1 + licenses/LICENSE-hubble-java17-third-party.txt + (EPL-2.0) org.glassfish.hk2:hk2-locator:2.6.1 + licenses/LICENSE-hubble-java17-third-party.txt + (EPL-2.0) org.glassfish.hk2:hk2-utils:2.6.1 + licenses/LICENSE-hubble-java17-third-party.txt + (EPL-2.0) org.glassfish.jersey.containers:jersey-container-servlet:2.34 + licenses/LICENSE-hubble-java17-third-party.txt + (EPL-2.0) org.glassfish.jersey.containers:jersey-container-servlet:3.1.11 + licenses/LICENSE-hubble-java17-third-party.txt + (EPL-2.0) org.glassfish.jersey.inject:jersey-hk2:2.34 + licenses/LICENSE-hubble-java17-third-party.txt + (WTFPL) org.reflections:reflections:0.10.2 + licenses/LICENSE-flink-final-third-party.txt + (WTFPL) org.reflections:reflections:0.9.12 + licenses/LICENSE-flink-final-third-party.txt diff --git a/hugegraph-dist/release-docs/NOTICE b/hugegraph-dist/release-docs/NOTICE index 54a3c1dba..3e8a3d110 100644 --- a/hugegraph-dist/release-docs/NOTICE +++ b/hugegraph-dist/release-docs/NOTICE @@ -5371,3 +5371,510 @@ Copyright 2001-2019 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). + +======================================================================== +Flink CDC runtime dependency notices +======================================================================== + +======================================================================== +Inherited shaded payload: jakarta.xml.bind:jakarta.xml.bind-api:2.3.2 / META-INF/NOTICE.md +======================================================================== + +# Notices for Eclipse Project for JAXB + +This content is produced and maintained by the Eclipse Project for JAXB project. + +* Project home: https://projects.eclipse.org/projects/ee4j.jaxb + +## Trademarks + +Eclipse Project for JAXB is a trademark of the Eclipse Foundation. + +## Copyright + +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. + +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Distribution License v. 1.0 which is available +at http://www.eclipse.org/org/documents/edl-v10.php. + +SPDX-License-Identifier: BSD-3-Clause + +## Source Code + +The project maintains the following source code repositories: + +* https://github.com/eclipse-ee4j/jaxb-api + +## Third-party Content + +This project leverages the following third party content. + +None + +## Cryptography + +Content may contain encryption software. The country in which you are currently +may have restrictions on the import, possession, and use, and/or re-export to +another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, +possession, or use, and re-export of encryption software, to see if this is +permitted. + +======================================================================== +Inherited shaded payload: commons-lang:commons-lang:2.6 / META-INF/NOTICE.txt +======================================================================== + +Apache Commons Lang +Copyright 2001-2011 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +======================================================================== +com.fasterxml.jackson.module:jackson-module-afterburner:2.21.4 / META-INF/NOTICE +======================================================================== + +# Jackson JSON processor + +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers, as well as supported +commercially by FasterXML.com. + +## Licensing + +Jackson core and extension components (as well their dependencies) may be licensed under +different licenses. +To find the details that apply to this artifact see the accompanying LICENSE file. +For more information, including possible other licensing options, contact +FasterXML.com (http://fasterxml.com). + +## Credits + +A list of contributors may be found from CREDITS file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. + +======================================================================== +io.swagger.core.v3:swagger-annotations:2.2.8 / META-INF/NOTICE +======================================================================== + +Swagger Core - swagger-annotations +Copyright (c) 2015. SmartBear Software Inc. +Swagger Core - swagger-annotations is licensed under Apache 2.0 license. +Copy of the Apache 2.0 license can be found in `LICENSE` file. + +======================================================================== +jakarta.ws.rs:jakarta.ws.rs-api:2.1.6 / META-INF/NOTICE.md +======================================================================== + +# Notices for the Jakarta RESTful Web Services Project + +This content is produced and maintained by the **Jakarta RESTful Web Services** +project. + +* Project home: https://projects.eclipse.org/projects/ee4j.jaxrs + +## Trademarks + +**Jakarta RESTful Web Services** is a trademark of the Eclipse Foundation. + +## Copyright + +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. + +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v. 2.0 which is available at +http://www.eclipse.org/legal/epl-2.0. This Source Code may also be made +available under the following Secondary Licenses when the conditions for such +availability set forth in the Eclipse Public License v. 2.0 are satisfied: GNU +General Public License, version 2 with the GNU Classpath Exception which is +available at https://www.gnu.org/software/classpath/license.html. + +SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + +## Source Code + +The project maintains the following source code repositories: + +* https://github.com/eclipse-ee4j/jaxrs-api + +## Third-party Content + +This project leverages the following third party content. + +javaee-api (7.0) + +* License: Apache-2.0 AND W3C + +JUnit (4.11) + +* License: Common Public License 1.0 + +Mockito (2.16.0) + +* Project: http://site.mockito.org +* Source: https://github.com/mockito/mockito/releases/tag/v2.16.0 + +## Cryptography + +Content may contain encryption software. The country in which you are currently +may have restrictions on the import, possession, and use, and/or re-export to +another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, +possession, or use, and re-export of encryption software, to see if this is +permitted. + + +======================================================================== +org.apache.flink:flink-cdc-common:3.6.0-2.2 / META-INF/NOTICE +======================================================================== + + +flink-cdc-common +Copyright 2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================== +org.apache.flink:flink-cdc-flink2-compat:3.6.0-2.2 / META-INF/NOTICE +======================================================================== + + +flink-cdc-flink2-compat +Copyright 2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================== +org.apache.flink:flink-cdc-runtime:3.6.0-2.2 / META-INF/NOTICE +======================================================================== + + +flink-cdc-runtime +Copyright 2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================== +org.apache.flink:flink-connector-debezium:3.6.0-2.2 / META-INF/NOTICE +======================================================================== + + +flink-connector-debezium +Copyright 2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================== +org.apache.flink:flink-connector-mysql-cdc:3.6.0-2.2 / META-INF/NOTICE +======================================================================== + + +flink-connector-mysql-cdc +Copyright 2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================== +org.apache.flink:flink-shaded-guava:31.1-jre-17.0 / META-INF/NOTICE +======================================================================== + +// ------------------------------------------------------------------ +// NOTICE file corresponding to the section 4d of The Apache License, +// Version 2.0, in this case for Apache Flink-shaded +// ------------------------------------------------------------------ + +Apache Flink-shaded +Copyright 2006-2023 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +flink-shaded-guava-30 +Copyright 2014-2021 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.google.guava:guava:31.1-jre +- com.google.guava:failureaccess:1.0.1 + + +======================================================================== +org.apache.flink:flink-shaded-guava:33.4.0-jre-20.0 / META-INF/NOTICE +======================================================================== + +// ------------------------------------------------------------------ +// NOTICE file corresponding to the section 4d of The Apache License, +// Version 2.0, in this case for Apache Flink-shaded +// ------------------------------------------------------------------ + +Apache Flink-shaded +Copyright 2006-2025 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +flink-shaded-guava-33 +Copyright 2014-2023 The Apache Software Foundation + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.google.guava:guava:33.4.0-jre +- com.google.guava:failureaccess:1.0.2 + + +======================================================================== +org.apache.kafka:connect-api:3.2.0 / NOTICE +org.apache.kafka:connect-file:3.2.0 / NOTICE +org.apache.kafka:connect-json:3.2.0 / NOTICE +org.apache.kafka:connect-runtime:3.2.0 / NOTICE +org.apache.kafka:connect-transforms:3.2.0 / NOTICE +org.apache.kafka:kafka-clients:3.2.0 / NOTICE +org.apache.kafka:kafka-tools:3.2.0 / NOTICE +======================================================================== + +Apache Kafka +Copyright 2022 The Apache Software Foundation. + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +This distribution has a binary dependency on jersey, which is available under the CDDL +License. The source code of jersey can be found at https://github.com/jersey/jersey/. + +This distribution has a binary test dependency on jqwik, which is available under +the Eclipse Public License 2.0. The source code can be found at +https://github.com/jlink/jqwik. + +The streams-scala (streams/streams-scala) module was donated by Lightbend and the original code was copyrighted by them: +Copyright (C) 2018 Lightbend Inc. +Copyright (C) 2017-2018 Alexis Seigneurin. + +This project contains the following code copied from Apache Hadoop: +clients/src/main/java/org/apache/kafka/common/utils/PureJavaCrc32C.java +Some portions of this file Copyright (c) 2004-2006 Intel Corporation and licensed under the BSD license. + +This project contains the following code copied from Apache Hive: +streams/src/main/java/org/apache/kafka/streams/state/internals/Murmur3.java + +======================================================================== +org.apache.kafka:connect-api:3.9.2 / NOTICE +org.apache.kafka:connect-file:3.9.2 / NOTICE +org.apache.kafka:connect-json:3.9.2 / NOTICE +org.apache.kafka:connect-runtime:3.9.2 / NOTICE +org.apache.kafka:connect-transforms:3.9.2 / NOTICE +org.apache.kafka:kafka-clients:3.9.2 / NOTICE +======================================================================== + +Apache Kafka +Copyright 2026 The Apache Software Foundation. + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +This distribution has a binary dependency on jersey, which is available under the CDDL +License. The source code of jersey can be found at https://github.com/jersey/jersey/. + +This distribution has a binary test dependency on jqwik, which is available under +the Eclipse Public License 2.0. The source code can be found at +https://github.com/jlink/jqwik. + +The streams-scala (streams/streams-scala) module was donated by Lightbend and the original code was copyrighted by them: +Copyright (C) 2018 Lightbend Inc. +Copyright (C) 2017-2018 Alexis Seigneurin. + +This project contains the following code copied from Apache Hadoop: +clients/src/main/java/org/apache/kafka/common/utils/PureJavaCrc32C.java +Some portions of this file Copyright (c) 2004-2006 Intel Corporation and licensed under the BSD license. + +This project contains the following code copied from Apache Hive: +streams/src/main/java/org/apache/kafka/streams/state/internals/Murmur3.java + +======================================================================== +org.apache.maven:maven-artifact:3.8.4 / META-INF/NOTICE +======================================================================== + + +Maven Artifact +Copyright 2001-2021 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================== +org.apache.maven:maven-artifact:3.9.6 / META-INF/NOTICE +======================================================================== + +Maven Artifact +Copyright 2001-2023 The Apache Software Foundation + + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +======================================================================== +org.codehaus.plexus:plexus-utils:3.3.0 / META-INF/NOTICE +org.codehaus.plexus:plexus-utils:3.5.1 / META-INF/NOTICE +======================================================================== + +This product includes software developed by the Indiana University + Extreme! Lab (http://www.extreme.indiana.edu/). + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +This product includes software developed by +ThoughtWorks (http://www.thoughtworks.com). + +This product includes software developed by +javolution (http://javolution.org/). + +This product includes software developed by +Rome (https://rome.dev.java.net/). + +======================================================================== +org.glassfish.jersey.containers:jersey-container-servlet:2.34 / META-INF/NOTICE.md +org.glassfish.jersey.inject:jersey-hk2:2.34 / META-INF/NOTICE.md +======================================================================== + +# Notice for Jersey +This content is produced and maintained by the Eclipse Jersey project. + +* Project home: https://projects.eclipse.org/projects/ee4j.jersey + +## Trademarks +Eclipse Jersey is a trademark of the Eclipse Foundation. + +## Copyright + +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. + +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v. 2.0 which is available at +http://www.eclipse.org/legal/epl-2.0. This Source Code may also be made +available under the following Secondary Licenses when the conditions for such +availability set forth in the Eclipse Public License v. 2.0 are satisfied: GNU +General Public License, version 2 with the GNU Classpath Exception which is +available at https://www.gnu.org/software/classpath/license.html. + +SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + +## Source Code +The project maintains the following source code repositories: + +* https://github.com/eclipse-ee4j/jersey + +## Third-party Content + +Angular JS, v1.6.6 +* License MIT (http://www.opensource.org/licenses/mit-license.php) +* Project: http://angularjs.org +* Coyright: (c) 2010-2017 Google, Inc. + +aopalliance Version 1 +* License: all the source code provided by AOP Alliance is Public Domain. +* Project: http://aopalliance.sourceforge.net +* Copyright: Material in the public domain is not protected by copyright + +Bean Validation API 2.0.2 +* License: Apache License, 2.0 +* Project: http://beanvalidation.org/1.1/ +* Copyright: 2009, Red Hat, Inc. and/or its affiliates, and individual contributors +* by the @authors tag. + +Hibernate Validator CDI, 6.1.2.Final +* License: Apache License, 2.0 +* Project: https://beanvalidation.org/ +* Repackaged in org.glassfish.jersey.server.validation.internal.hibernate + +Bootstrap v3.3.7 +* License: MIT license (https://github.com/twbs/bootstrap/blob/master/LICENSE) +* Project: http://getbootstrap.com +* Copyright: 2011-2016 Twitter, Inc + +Google Guava Version 18.0 +* License: Apache License, 2.0 +* Copyright (C) 2009 The Guava Authors + +javax.inject Version: 1 +* License: Apache License, 2.0 +* Copyright (C) 2009 The JSR-330 Expert Group + +Javassist Version 3.25.0-GA +* License: Apache License, 2.0 +* Project: http://www.javassist.org/ +* Copyright (C) 1999- Shigeru Chiba. All Rights Reserved. + +Jackson JAX-RS Providers Version 2.10.1 +* License: Apache License, 2.0 +* Project: https://github.com/FasterXML/jackson-jaxrs-providers +* Copyright: (c) 2009-2011 FasterXML, LLC. All rights reserved unless otherwise indicated. + +jQuery v1.12.4 +* License: jquery.org/license +* Project: jquery.org +* Copyright: (c) jQuery Foundation + +jQuery Barcode plugin 0.3 +* License: MIT & GPL (http://www.opensource.org/licenses/mit-license.php & http://www.gnu.org/licenses/gpl.html) +* Project: http://www.pasella.it/projects/jQuery/barcode +* Copyright: (c) 2009 Antonello Pasella antonello.pasella@gmail.com + +JSR-166 Extension - JEP 266 +* License: CC0 +* No copyright +* Written by Doug Lea with assistance from members of JCP JSR-166 Expert Group and released to the public domain, as explained at http://creativecommons.org/publicdomain/zero/1.0/ + +KineticJS, v4.7.1 +* License: MIT license (http://www.opensource.org/licenses/mit-license.php) +* Project: http://www.kineticjs.com, https://github.com/ericdrowell/KineticJS +* Copyright: Eric Rowell + +org.objectweb.asm Version 8.0 +* License: Modified BSD (http://asm.objectweb.org/license.html) +* Copyright (c) 2000-2011 INRIA, France Telecom. All rights reserved. + +org.osgi.core version 6.0.0 +* License: Apache License, 2.0 +* Copyright (c) OSGi Alliance (2005, 2008). All Rights Reserved. + +org.glassfish.jersey.server.internal.monitoring.core +* License: Apache License, 2.0 +* Copyright (c) 2015-2018 Oracle and/or its affiliates. All rights reserved. +* Copyright 2010-2013 Coda Hale and Yammer, Inc. + +W3.org documents +* License: W3C License +* Copyright: Copyright (c) 1994-2001 World Wide Web Consortium, (Massachusetts Institute of Technology, Institut National de Recherche en Informatique et en Automatique, Keio University). All Rights Reserved. http://www.w3.org/Consortium/Legal/ diff --git a/hugegraph-dist/release-docs/licenses/LICENSE-flink-final-third-party.txt b/hugegraph-dist/release-docs/licenses/LICENSE-flink-final-third-party.txt new file mode 100644 index 000000000..a19373df9 --- /dev/null +++ b/hugegraph-dist/release-docs/licenses/LICENSE-flink-final-third-party.txt @@ -0,0 +1,1154 @@ +Complete additional third-party license texts for the Flink CDC runtime. + +Exact resolved GAV/ZIP resources and release sources identify each grant. +Identical full texts are reproduced once. Notices appear in the main NOTICE. +The Zstandard native library uses its BSD option. Eclipse components use +the EPL-2.0 option, Jetty uses Apache-2.0, and Reflections uses WTFPL. +The inherited shade carries additional baseline resources (including the +Arrow Apache header template); their original full text is retained below. +They are coverage repairs for existing dependency identities, not CDC additions. +Existing complete terms remain in the component files referenced by LICENSE. + +======================================================================== +Inherited shaded payload: org.apache.arrow:arrow-vector:2.0.0 / codegen/includes/license.ftl +======================================================================== + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +======================================================================== +Inherited shaded payload: io.grpc:grpc-netty-shaded:1.39.0 / META-INF/license/LICENSE.boringssl.txt +======================================================================== + +BoringSSL is a fork of OpenSSL. As such, large parts of it fall under OpenSSL +licensing. Files that are completely new have a Google copyright and an ISC +license. This license is reproduced at the bottom of this file. +Contributors to BoringSSL are required to follow the CLA rules for Chromium: +https://cla.developers.google.com/clas +Files in third_party/ have their own licenses, as described therein. The MIT +license, for third_party/fiat, which, unlike other third_party directories, is +compiled into non-test libraries, is included below. +The OpenSSL toolkit stays under a dual license, i.e. both the conditions of the +OpenSSL License and the original SSLeay license apply to the toolkit. See below +for the actual license texts. Actually both licenses are BSD-style Open Source +licenses. In case of any license issues related to OpenSSL please contact +openssl-core@openssl.org. +The following are Google-internal bug numbers where explicit permission from +some authors is recorded for use of their work. (This is purely for our own +record keeping.) + 27287199 + 27287880 + 27287883 + OpenSSL License + --------------- +/* ==================================================================== + * Copyright (c) 1998-2011 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + Original SSLeay License + ----------------------- +/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) + * All rights reserved. + * + * This package is an SSL implementation written + * by Eric Young (eay@cryptsoft.com). + * The implementation was written so as to conform with Netscapes SSL. + * + * This library is free for commercial and non-commercial use as long as + * the following conditions are aheared to. The following conditions + * apply to all code found in this distribution, be it the RC4, RSA, + * lhash, DES, etc., code; not just the SSL code. The SSL documentation + * included with this distribution is covered by the same copyright terms + * except that the holder is Tim Hudson (tjh@cryptsoft.com). + * + * Copyright remains Eric Young's, and as such any Copyright notices in + * the code are not to be removed. + * If this package is used in a product, Eric Young should be given attribution + * as the author of the parts of the library used. + * This can be in the form of a textual message at program startup or + * in documentation (online or textual) provided with the package. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * "This product includes cryptographic software written by + * Eric Young (eay@cryptsoft.com)" + * The word 'cryptographic' can be left out if the rouines from the library + * being used are not cryptographic related :-). + * 4. If you include any Windows specific code (or a derivative thereof) from + * the apps directory (application code) you must include an acknowledgement: + * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" + * + * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * The licence and distribution terms for any publically available version or + * derivative of this code cannot be changed. i.e. this code cannot simply be + * copied and put under another distribution licence + * [including the GNU Public Licence.] + */ +ISC license used for completely new code in BoringSSL: +/* Copyright (c) 2015, Google Inc. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION + * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN + * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ +The code in third_party/fiat carries the MIT license: +Copyright (c) 2015-2016 the fiat-crypto authors (see +https://github.com/mit-plv/fiat-crypto/blob/master/AUTHORS). +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +Licenses for support code +------------------------- +Parts of the TLS test suite are under the Go license. This code is not included +in BoringSSL (i.e. libcrypto and libssl) when compiled, however, so +distributing code linked against BoringSSL does not trigger this license: +Copyright (c) 2009 The Go Authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +BoringSSL uses the Chromium test infrastructure to run a continuous build, +trybots etc. The scripts which manage this, and the script for generating build +metadata, are under the Chromium license. Distributing code linked against +BoringSSL does not trigger this license. +Copyright 2015 The Chromium Authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +======================================================================== +Inherited shaded payload: io.grpc:grpc-netty-shaded:1.39.0 / META-INF/license/LICENSE.aix-netbsd.txt +======================================================================== + + + LICENSE ISSUES + ============== + + The OpenSSL toolkit stays under a double license, i.e. both the conditions of + the OpenSSL License and the original SSLeay license apply to the toolkit. + See below for the actual license texts. + + OpenSSL License + --------------- + +/* ==================================================================== + * Copyright (c) 1998-2019 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + + Original SSLeay License + ----------------------- + +/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) + * All rights reserved. + * + * This package is an SSL implementation written + * by Eric Young (eay@cryptsoft.com). + * The implementation was written so as to conform with Netscapes SSL. + * + * This library is free for commercial and non-commercial use as long as + * the following conditions are aheared to. The following conditions + * apply to all code found in this distribution, be it the RC4, RSA, + * lhash, DES, etc., code; not just the SSL code. The SSL documentation + * included with this distribution is covered by the same copyright terms + * except that the holder is Tim Hudson (tjh@cryptsoft.com). + * + * Copyright remains Eric Young's, and as such any Copyright notices in + * the code are not to be removed. + * If this package is used in a product, Eric Young should be given attribution + * as the author of the parts of the library used. + * This can be in the form of a textual message at program startup or + * in documentation (online or textual) provided with the package. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * "This product includes cryptographic software written by + * Eric Young (eay@cryptsoft.com)" + * The word 'cryptographic' can be left out if the rouines from the library + * being used are not cryptographic related :-). + * 4. If you include any Windows specific code (or a derivative thereof) from + * the apps directory (application code) you must include an acknowledgement: + * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" + * + * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * The licence and distribution terms for any publically available version or + * derivative of this code cannot be changed. i.e. this code cannot simply be + * copied and put under another distribution licence + * [including the GNU Public Licence.] + */ + + +======================================================================== +Inherited shaded payload: org.codehaus.groovy:groovy-all:2.4.21 / META-INF/licenses/antlr2-license.txt +======================================================================== + +ANTLR 2 License + +We reserve no legal rights to the ANTLR--it is fully in the public domain. An individual or company may do +whatever they wish with source code distributed with ANTLR or the code generated by ANTLR, including the +incorporation of ANTLR, or its output, into commerical software. + +We encourage users to develop software with ANTLR. However, we do ask that credit is given to us for +developing ANTLR. By "credit", we mean that if you use ANTLR or incorporate any source code into one of your +programs (commercial product, research project, or otherwise) that you acknowledge this fact somewhere in +the documentation, research report, etc... If you like ANTLR and have developed a nice tool with the output, +please mention that you developed it using ANTLR. In addition, we ask that the headers remain intact in our +source code. As long as these guidelines are kept, we expect to continue enhancing this system and expect to +make other tools available as they are completed. + +In countries where the Public Domain status of the work may not be valid, the author grants a copyright +licence to the general public to deal in the work without restriction and permission to sublicence derivates +under the terms of any (OSI approved) Open Source licence. + +The Python parser generator code under antlr/actions/python/ is covered by the 3-clause BSD licence (this +part is included in the binary JAR files); the run-time part under lib/python/ is covered by the GNU GPL, +version 3 or later (this part is not included in the binary JAR files). See [1] for the full details. + +https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=750643#80%22 + +======================================================================== +Inherited shaded payload: org.codehaus.groovy:groovy-all:2.4.21 / META-INF/licenses/jline2-license.txt +======================================================================== + +Copyright (c) 2002-2012, the original author or authors. +All rights reserved. + +http://www.opensource.org/licenses/bsd-license.php + +Redistribution and use in source and binary forms, with or +without modification, are permitted provided that the following +conditions are met: + +Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with +the distribution. + +Neither the name of JLine nor the names of its contributors +may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, +BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, +OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE. + +======================================================================== +Inherited shaded payload: org.codehaus.groovy:groovy-all:2.4.21 / META-INF/licenses/jsr223-license.txt +======================================================================== + +Copyright (c) 2006, Sun Microsystems, Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + - Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + - Neither the name of the Sun Microsystems, Inc. nor the names of + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. + +======================================================================== +Inherited shaded payload: org.codehaus.groovy:groovy-all:2.4.21 / META-INF/licenses/normalize-stylesheet-license.txt +======================================================================== + +Copyright (c) Nicolas Gallagher and Jonathan Neal + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +======================================================================== +Inherited shaded payload: org.postgresql:postgresql:42.4.1 / META-INF/licenses/com.ongres.scram/client-2.1/LICENSE +Inherited shaded payload: org.postgresql:postgresql:42.4.1 / META-INF/licenses/com.ongres.scram/common-2.1/LICENSE +======================================================================== + +Copyright 2017, OnGres. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +======================================================================== +Inherited shaded payload: org.postgresql:postgresql:42.4.1 / META-INF/licenses/com.ongres.stringprep/saslprep-1.1/LICENSE +Inherited shaded payload: org.postgresql:postgresql:42.4.1 / META-INF/licenses/com.ongres.stringprep/stringprep-1.1/LICENSE +======================================================================== + +Copyright 2019, OnGres. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +======================================================================== +Inherited shaded payload: org.datanucleus:datanucleus-core:4.1.17 / META-INF/ASM_LICENSE.txt +======================================================================== + +ObjectWeb ASM 4.1 is repackaged in this datanucleus-core jar under +org.datanucleus.asm. The files in that package are licensed under +the following terms +===================================================================== +Copyright (c) 2000-2011 INRIA, France Telecom +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. +===================================================================== + +======================================================================== +com.fasterxml.jackson.module:jackson-module-afterburner:2.21.4 / META-INF/LICENSE +======================================================================== + +This copy of Jackson JSON processor `jackson-module-afterburner` module is licensed under the +Apache (Software) License, version 2.0 ("the License"). +See the License for details about distribution rights, and the +specific rights regarding derivative works. + +You may obtain a copy of the License at: + + http://www.apache.org/licenses/LICENSE-2.0 + +Additional licensing information exists for following 3rd party library dependencies + +### ASM + +ASM: a very small and fast Java bytecode manipulation framework +Copyright (c) 2000-2011 INRIA, France Telecom +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. + + +======================================================================== +io.swagger.core.v3:swagger-annotations:2.2.8 / META-INF/LICENSE +======================================================================== + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright (c) 2015. SmartBear Software Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +======================================================================== +org.codehaus.plexus:plexus-utils:3.3.0 / licenses/extreme.indiana.edu.license.TXT +org.codehaus.plexus:plexus-utils:3.5.1 / licenses/extreme.indiana.edu.license.TXT +======================================================================== + +Indiana University Extreme! Lab Software License + +Version 1.1.1 + +Copyright (c) 2002 Extreme! Lab, Indiana University. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution. + +3. The end-user documentation included with the redistribution, if any, + must include the following acknowledgment: + + "This product includes software developed by the Indiana University + Extreme! Lab (http://www.extreme.indiana.edu/)." + +Alternately, this acknowledgment may appear in the software itself, +if and wherever such third-party acknowledgments normally appear. + +4. The names "Indiana University" and "Indiana University Extreme! Lab" +must not be used to endorse or promote products derived from this +software without prior written permission. For written permission, +please contact http://www.extreme.indiana.edu/. + +5. Products derived from this software may not use "Indiana University" +name nor may "Indiana University" appear in their name, without prior +written permission of the Indiana University. + +THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESSED OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHORS, COPYRIGHT HOLDERS OR ITS CONTRIBUTORS +BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +======================================================================== +org.codehaus.plexus:plexus-utils:3.3.0 / licenses/javolution.license.TXT +org.codehaus.plexus:plexus-utils:3.5.1 / licenses/javolution.license.TXT +======================================================================== + +Javolution - Java(TM) Solution for Real-Time and Embedded Systems +Copyright (c) 2006, Javolution (http://javolution.org) +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +======================================================================== +https://raw.githubusercontent.com/spdx/license-list-data/main/text/CDDL-1.0.txt +======================================================================== + +COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) +Version 1.0 + +1. Definitions. + +1.1. “Contributor” means each individual or entity that creates or contributes to the creation of Modifications. + +1.2. “Contributor Version” means the combination of the Original Software, prior Modifications used by a Contributor (if any), and the Modifications made by that particular Contributor. + +1.3. “Covered Software” means (a) the Original Software, or (b) Modifications, or (c) the combination of files containing Original Software with files containing Modifications, in each case including portions thereof. + +1.4. “Executable” means the Covered Software in any form other than Source Code. + +1.5. “Initial Developer” means the individual or entity that first makes Original Software available under this License. + +1.6. “Larger Work” means a work which combines Covered Software or portions thereof with code not governed by the terms of this License. + +1.7. “License” means this document. + +1.8. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently acquired, any and all of the rights conveyed herein. + +1.9. “Modifications” means the Source Code and Executable form of any of the following: + + A. Any file that results from an addition to, deletion from or modification of the contents of a file containing Original Software or previous Modifications; + + B. Any new file that contains any part of the Original Software or previous Modification; or + + C. Any new file that is contributed or otherwise made available under the terms of this License. + +1.10. “Original Software” means the Source Code and Executable form of computer software code that is originally released under this License. + +1.11. “Patent Claims” means any patent claim(s), now owned or hereafter acquired, including without limitation, method, process, and apparatus claims, in any patent Licensable by grantor. + +1.12. “Source Code” means (a) the common form of computer software code in which modifications are made and (b) associated documentation included in or with such code. + +1.13. “You” (or “Your”) means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, “You” includes any entity which controls, is controlled by, or is under common control with You. For purposes of this definition, “control” means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of more than fifty percent (50%) of the outstanding shares or beneficial ownership of such entity. + +2. License Grants. + +2.1. The Initial Developer Grant. +Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, the Initial Developer hereby grants You a world-wide, royalty-free, non-exclusive license: + + (a) under intellectual property rights (other than patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, display, perform, sublicense and distribute the Original Software (or portions thereof), with or without Modifications, and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using or selling of Original Software, to make, have made, use, practice, sell, and offer for sale, and/or otherwise dispose of the Original Software (or portions thereof). + + (c) The licenses granted in Sections 2.1(a) and (b) are effective on the date Initial Developer first distributes or otherwise makes the Original Software available to a third party under the terms of this License. + + (d) Notwithstanding Section 2.1(b) above, no patent license is granted: (1) for code that You delete from the Original Software, or (2) for infringements caused by: (i) the modification of the Original Software, or (ii) the combination of the Original Software with other software or devices. + +2.2. Contributor Grant. +Conditioned upon Your compliance with Section 3.1 below and subject to third party intellectual property claims, each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license: + + (a) under intellectual property rights (other than patent or trademark) Licensable by Contributor to use, reproduce, modify, display, perform, sublicense and distribute the Modifications created by such Contributor (or portions thereof), either on an unmodified basis, with other Modifications, as Covered Software and/or as part of a Larger Work; and + + (b) under Patent Claims infringed by the making, using, or selling of Modifications made by that Contributor either alone and/or in combination with its Contributor Version (or portions of such combination), to make, use, sell, offer for sale, have made, and/or otherwise dispose of: (1) Modifications made by that Contributor (or portions thereof); and (2) the combination of Modifications made by that Contributor with its Contributor Version (or portions of such combination). + + (c) The licenses granted in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first distributes or otherwise makes the Modifications available to a third party. + + (d) Notwithstanding Section 2.2(b) above, no patent license is granted: (1) for any code that Contributor has deleted from the Contributor Version; (2) for infringements caused by: (i) third party modifications of Contributor Version, or (ii) the combination of Modifications made by that Contributor with other software (except as part of the Contributor Version) or other devices; or (3) under Patent Claims infringed by Covered Software in the absence of Modifications made by that Contributor. + +3. Distribution Obligations. + +3.1. Availability of Source Code. +Any Covered Software that You distribute or otherwise make available in Executable form must also be made available in Source Code form and that Source Code form must be distributed only under the terms of this License. You must include a copy of this License with every copy of the Source Code form of the Covered Software You distribute or otherwise make available. You must inform recipients of any such Covered Software in Executable form as to how they can obtain such Covered Software in Source Code form in a reasonable manner on or through a medium customarily used for software exchange. + +3.2. Modifications. +The Modifications that You create or to which You contribute are governed by the terms of this License. You represent that You believe Your Modifications are Your original creation(s) and/or You have sufficient rights to grant the rights conveyed by this License. + +3.3. Required Notices. +You must include a notice in each of Your Modifications that identifies You as the Contributor of the Modification. You may not remove or alter any copyright, patent or trademark notices contained within the Covered Software, or any notices of licensing or any descriptive text giving attribution to any Contributor or the Initial Developer. + +3.4. Application of Additional Terms. +You may not offer or impose any terms on any Covered Software in Source Code form that alters or restricts the applicable version of this License or the recipients’ rights hereunder. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, you may do so only on Your own behalf, and not on behalf of the Initial Developer or any Contributor. You must make it absolutely clear that any such warranty, support, indemnity or liability obligation is offered by You alone, and You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of warranty, support, indemnity or liability terms You offer. + +3.5. Distribution of Executable Versions. +You may distribute the Executable form of the Covered Software under the terms of this License or under the terms of a license of Your choice, which may contain terms different from this License, provided that You are in compliance with the terms of this License and that the license for the Executable form does not attempt to limit or alter the recipient’s rights in the Source Code form from the rights set forth in this License. If You distribute the Covered Software in Executable form under a different license, You must make it absolutely clear that any terms which differ from this License are offered by You alone, not by the Initial Developer or Contributor. You hereby agree to indemnify the Initial Developer and every Contributor for any liability incurred by the Initial Developer or such Contributor as a result of any such terms You offer. + +3.6. Larger Works. +You may create a Larger Work by combining Covered Software with other code not governed by the terms of this License and distribute the Larger Work as a single product. In such a case, You must make sure the requirements of this License are fulfilled for the Covered Software. + +4. Versions of the License. + +4.1. New Versions. +Sun Microsystems, Inc. is the initial license steward and may publish revised and/or new versions of this License from time to time. Each version will be given a distinguishing version number. Except as provided in Section 4.3, no one other than the license steward has the right to modify this License. + +4.2. Effect of New Versions. +You may always continue to use, distribute or otherwise make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. If the Initial Developer includes a notice in the Original Software prohibiting it from being distributed or otherwise made available under any subsequent version of the License, You must distribute and make the Covered Software available under the terms of the version of the License under which You originally received the Covered Software. Otherwise, You may also choose to use, distribute or otherwise make the Covered Software available under the terms of any subsequent version of the License published by the license steward. + +4.3. Modified Versions. +When You are an Initial Developer and You want to create a new license for Your Original Software, You may create and use a modified version of this License if You: (a) rename the license and remove any references to the name of the license steward (except to note that the license differs from this License); and (b) otherwise make it clear that the license contains terms which differ from this License. + +5. DISCLAIMER OF WARRANTY. + +COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. + +6. TERMINATION. + +6.1. This License and the rights granted hereunder will terminate automatically if You fail to comply with terms herein and fail to cure such breach within 30 days of becoming aware of the breach. Provisions which, by their nature, must remain in effect beyond the termination of this License shall survive. + +6.2. If You assert a patent infringement claim (excluding declaratory judgment actions) against Initial Developer or a Contributor (the Initial Developer or Contributor against whom You assert such claim is referred to as “Participant”) alleging that the Participant Software (meaning the Contributor Version where the Participant is a Contributor or the Original Software where the Participant is the Initial Developer) directly or indirectly infringes any patent, then any and all rights granted directly or indirectly to You by such Participant, the Initial Developer (if the Initial Developer is not the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from Participant terminate prospectively and automatically at the expiration of such 60 day notice period, unless if within such 60 day period You withdraw Your claim with respect to the Participant Software against such Participant either unilaterally or pursuant to a written agreement with Participant. + +6.3. In the event of termination under Sections 6.1 or 6.2 above, all end user licenses that have been validly granted by You or any distributor hereunder prior to termination (excluding licenses granted to You by any distributor) shall survive termination. + +7. LIMITATION OF LIABILITY. + +UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTY’S NEGLIGENCE TO THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU. + +8. U.S. GOVERNMENT END USERS. + +The Covered Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 (Oct. 1995), consisting of “commercial computer software” (as that term is defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software with only those rights set forth herein. This U.S. Government Rights clause is in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision that addresses Government rights in computer software under this License. + +9. MISCELLANEOUS. + +This License represents the complete agreement concerning subject matter hereof. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. This License shall be governed by the law of the jurisdiction specified in a notice contained within the Original Software (except to the extent applicable law, if any, provides otherwise), excluding such jurisdiction’s conflict-of-law provisions. Any litigation relating to this License shall be subject to the jurisdiction of the courts located in the jurisdiction and venue specified in a notice contained within the Original Software, with the losing party responsible for costs, including, without limitation, court costs and reasonable attorneys’ fees and expenses. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not apply to this License. You agree that You alone are responsible for compliance with the United States export administration regulations (and the export control laws and regulation of any other countries) when You use, distribute or otherwise make available any Covered Software. + +10. RESPONSIBILITY FOR CLAIMS. + +As between Initial Developer and the Contributors, each party is responsible for claims and damages arising, directly or indirectly, out of its utilization of rights under this License and You agree to work with Initial Developer and Contributors to distribute such responsibility on an equitable basis. Nothing herein is intended or shall be deemed to constitute any admission of liability. + +======================================================================== +https://raw.githubusercontent.com/argparse4j/argparse4j/argparse4j-0.7.0/LICENSE.txt +======================================================================== + +/* + * Copyright (C) 2011, 2014, 2015 Tatsuhiro Tsujikawa + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, copy, + * modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +======================================================================== +https://raw.githubusercontent.com/ronmamo/reflections/0.9.12/COPYING.txt +https://raw.githubusercontent.com/ronmamo/reflections/0.10.2/COPYING.txt +======================================================================== + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + Version 2, December 2004 + + Copyright (C) 2004 Sam Hocevar + + Everyone is permitted to copy and distribute verbatim or modified + copies of this license document, and changing it is allowed as long + as the name is changed. + + DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. You just DO WHAT THE FUCK YOU WANT TO. + + +======================================================================== +https://raw.githubusercontent.com/luben/zstd-jni/v1.5.0-2/src/main/native/LICENSE +https://raw.githubusercontent.com/luben/zstd-jni/v1.5.0-4/src/main/native/LICENSE +======================================================================== + +BSD License + +For Zstandard software + +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/hugegraph-dist/scripts/dependency/known-dependencies.txt b/hugegraph-dist/scripts/dependency/known-dependencies.txt index 1e8dce0c0..e7e022733 100644 --- a/hugegraph-dist/scripts/dependency/known-dependencies.txt +++ b/hugegraph-dist/scripts/dependency/known-dependencies.txt @@ -1,8 +1,10 @@ +HikariCP-4.0.3.jar HikariCP-6.3.3.jar HikariCP-java7-2.4.12.jar ST4-4.0.4.jar accessors-smart-2.4.2.jar accessors-smart-2.5.2.jar +activation-1.1.1.jar aircompressor-0.10.jar aircompressor-0.21.jar angus-activation-2.0.3.jar @@ -15,11 +17,14 @@ ansj_seg-5.1.6.jar ant-1.9.1.jar ant-launcher-1.9.1.jar antlr-runtime-3.5.2.jar +antlr4-runtime-4.8.jar antlr4-runtime-4.9.1.jar aopalliance-1.0.jar +aopalliance-repackaged-2.6.1.jar aopalliance-repackaged-3.0.6.jar apache-curator-2.12.0.pom apiguardian-api-1.1.2.jar +argparse4j-0.7.0.jar arrow-format-0.8.0.jar arrow-format-2.0.0.jar arrow-memory-0.8.0.jar @@ -67,6 +72,16 @@ commons-math3-3.4.1.jar commons-net-3.9.0.jar commons-pool-1.6.jar commons-text-1.10.0.jar +connect-api-3.2.0.jar +connect-api-3.9.2.jar +connect-file-3.2.0.jar +connect-file-3.9.2.jar +connect-json-3.2.0.jar +connect-json-3.9.2.jar +connect-runtime-3.2.0.jar +connect-runtime-3.9.2.jar +connect-transforms-3.2.0.jar +connect-transforms-3.9.2.jar cucumber-core-7.21.1.jar cucumber-expressions-18.0.1.jar cucumber-gherkin-7.21.1.jar @@ -82,6 +97,11 @@ curator-recipes-2.13.0.jar curator-recipes-4.2.0.jar datanucleus-core-4.1.17.jar datatable-7.21.1.jar +debezium-api-1.9.8.Final.jar +debezium-connector-mysql-1.9.8.Final.jar +debezium-core-1.9.8.Final.jar +debezium-ddl-parser-1.9.8.Final.jar +debezium-embedded-1.9.8.Final.jar disruptor-3.3.6.jar disruptor-3.3.7.jar dnsjava-2.1.7.jar @@ -94,7 +114,7 @@ elasticsearch-java-7.16.1.jar elasticsearch-rest-client-7.16.1.jar error_prone_annotations-2.3.3.jar error_prone_annotations-2.3.4.jar -esri-geometry-api-2.0.0.jar +esri-geometry-api-2.2.0.jar exp4j-0.4.8.jar failsafe-2.4.1.jar failureaccess-1.0.1.jar @@ -102,6 +122,13 @@ fastutil-8.5.9.jar findbugs-annotations-1.3.9-1.jar flatbuffers-1.2.0-3f79e055.jar flatbuffers-java-1.9.0.jar +flink-cdc-common-3.6.0-2.2.jar +flink-cdc-flink2-compat-3.6.0-2.2.jar +flink-cdc-runtime-3.6.0-2.2.jar +flink-connector-debezium-3.6.0-2.2.jar +flink-connector-mysql-cdc-3.6.0-2.2.jar +flink-shaded-guava-31.1-jre-17.0.jar +flink-shaded-guava-33.4.0-jre-20.0.jar fst-2.50.jar fury-core-0.9.0.jar generex-1.0.2.jar @@ -199,8 +226,11 @@ hive-storage-api-2.7.0.jar hive-storage-api-2.7.2.jar hive-upgrade-acid-3.1.3.jar hive-vector-code-gen-3.1.3.jar +hk2-api-2.6.1.jar hk2-api-3.0.6.jar +hk2-locator-2.6.1.jar hk2-locator-3.0.6.jar +hk2-utils-2.6.1.jar hk2-utils-3.0.6.jar hppc-0.7.1.jar hppc-0.7.2.jar @@ -232,7 +262,9 @@ jackson-databind-2.12.3.jar jackson-databind-2.15.2.jar jackson-databind-2.21.4.jar jackson-dataformat-yaml-2.21.4.jar +jackson-datatype-jdk8-2.12.3.jar jackson-datatype-jdk8-2.21.4.jar +jackson-datatype-jsr310-2.12.3.jar jackson-datatype-jsr310-2.21.4.jar jackson-jaxrs-1.9.2.jar jackson-jaxrs-base-2.12.3.jar @@ -242,6 +274,7 @@ jackson-jaxrs-json-provider-2.12.3.jar jackson-jaxrs-json-provider-2.15.2.jar jackson-jaxrs-json-provider-2.21.4.jar jackson-mapper-asl-1.9.13.jar +jackson-module-afterburner-2.21.4.jar jackson-module-jaxb-annotations-2.12.3.jar jackson-module-jaxb-annotations-2.15.2.jar jackson-module-jaxb-annotations-2.21.4.jar @@ -252,8 +285,10 @@ jakarta.activation-api-1.2.1.jar jakarta.activation-api-1.2.2.jar jakarta.activation-api-2.1.4.jar jakarta.annotation-api-2.1.1.jar +jakarta.inject-2.6.1.jar jakarta.inject-api-2.0.1.jar jakarta.json-2.0.1.jar +jakarta.ws.rs-api-2.1.6.jar jakarta.ws.rs-api-3.1.0.jar jakarta.xml.bind-api-2.3.2.jar jakarta.xml.bind-api-2.3.3.jar @@ -276,6 +311,7 @@ javax.json-1.0.jar javax.servlet-api-3.1.0.jar javax.servlet.jsp-2.3.2.jar javax.servlet.jsp-api-2.3.1.jar +javax.ws.rs-api-2.1.1.jar javolution-5.5.1.jar jaxb-api-2.2.11.jar jaxb-api-2.2.12.jar @@ -295,10 +331,13 @@ jersey-apache-connector-3.1.11.jar jersey-client-1.19.jar jersey-client-3.1.11.jar jersey-common-3.1.11.jar +jersey-container-servlet-2.34.jar +jersey-container-servlet-3.1.11.jar jersey-container-servlet-core-2.25.1.jar jersey-container-servlet-core-3.1.11.jar jersey-core-1.19.jar jersey-guice-1.19.jar +jersey-hk2-2.34.jar jersey-hk2-3.1.11.jar jersey-json-1.19.jar jersey-server-1.19.jar @@ -306,8 +345,12 @@ jersey-servlet-1.19.jar jetcd-common-0.5.9.jar jetcd-core-0.5.9.jar jettison-1.1.jar -jetty-client-9.4.33.v20201020.jar +jetty-alpn-client-12.0.36.jar +jetty-client-12.0.36.jar jetty-client-9.4.40.v20210413.jar +jetty-client-9.4.44.v20210927.jar +jetty-continuation-9.4.44.v20210927.jar +jetty-continuation-9.4.57.v20241219.jar jetty-http-12.0.36.jar jetty-http-9.4.40.v20210413.jar jetty-io-12.0.36.jar @@ -319,6 +362,8 @@ jetty-security-9.4.40.v20210413.jar jetty-server-12.0.36.jar jetty-server-9.4.40.v20210413.jar jetty-servlet-9.4.40.v20210413.jar +jetty-servlets-9.4.44.v20210927.jar +jetty-servlets-9.4.57.v20241219.jar jetty-util-12.0.36.jar jetty-util-9.4.40.v20210413.jar jetty-util-ajax-12.0.36.jar @@ -336,6 +381,8 @@ jline-3.9.0.jar jna-5.12.1.jar joda-time-2.10.8.jar joni-2.1.11.jar +jose4j-0.7.9.jar +jose4j-0.9.6.jar jpam-1.1.jar jraft-core-1.3.14.jar jsch-0.1.55.jar @@ -351,6 +398,9 @@ jsr305-3.0.2.jar jsr311-api-1.1.1.jar jul-to-slf4j-2.0.18.jar junit-xml-formatter-0.7.1.jar +kafka-clients-3.2.0.jar +kafka-clients-3.9.2.jar +kafka-tools-3.2.0.jar kerb-admin-1.0.1.jar kerb-admin-2.0.1.jar kerb-client-1.0.1.jar @@ -428,7 +478,10 @@ lucene-core-8.11.2.jar lucene-queries-4.7.2.jar lucene-queryparser-4.7.2.jar lucene-sandbox-4.7.2.jar +lz4-java-1.10.1.jar lz4-java-1.4.0.jar +maven-artifact-3.8.4.jar +maven-artifact-3.9.6.jar memory-0.9.0.jar messages-27.2.0.jar metrics-core-4.2.0.jar @@ -451,6 +504,7 @@ mybatis-plus-spring-boot-autoconfigure-3.5.17.jar mybatis-plus-spring-boot-native-image-3.5.17.jar mybatis-plus-spring-boot3-starter-3.5.17.jar mybatis-spring-3.0.5.jar +mysql-binlog-connector-java-0.27.2.jar netty-3.10.6.Final.jar netty-all-4.1.135.Final.jar netty-all-4.1.65.Final.jar @@ -518,6 +572,8 @@ parboiled-core-1.1.8.jar parboiled-core-1.2.0.jar parquet-hadoop-bundle-1.10.0.jar perfmark-api-0.23.0.jar +plexus-utils-3.3.0.jar +plexus-utils-3.5.1.jar postgresql-42.4.1.jar postgresql-42.7.11.jar proto-google-common-protos-2.0.1.jar @@ -528,6 +584,10 @@ protobuf-java-3.19.6.jar protobuf-java-util-3.17.2.jar query-13.6.0.jar re2j-1.1.jar +reflections-0.10.2.jar +reflections-0.9.12.jar +reload4j-1.2.19.jar +reload4j-1.2.25.jar sfm-converter-8.2.1.jar sfm-csv-8.2.1.jar sfm-map-8.2.1.jar @@ -567,6 +627,7 @@ spring-web-6.2.19.jar spring-webmvc-6.2.19.jar stax-api-1.0.1.jar stax2-api-4.2.1.jar +swagger-annotations-2.2.8.jar tag-expressions-6.1.2.jar testng-xml-formatter-0.3.1.jar threeten-extra-1.5.0.jar @@ -585,3 +646,5 @@ xz-1.8.jar zjsonpatch-0.3.0.jar zookeeper-3.6.2.jar zookeeper-jute-3.6.2.jar +zstd-jni-1.5.0-2.jar +zstd-jni-1.5.0-4.jar diff --git a/hugegraph-loader/README.md b/hugegraph-loader/README.md index 9b660f792..09a090717 100644 --- a/hugegraph-loader/README.md +++ b/hugegraph-loader/README.md @@ -131,3 +131,5 @@ Server 1.7 compatibility CI uses the pinned official ASF 1.7.0 binary archive. T ## 5. License hugegraph-loader is licensed under Apache 2.0 License. + +See the [Flink CDC Java 17 validation](docs/flink-cdc-java17.md) for its current limits and launcher syntax. diff --git a/hugegraph-loader/assembly/descriptor/assembly.xml b/hugegraph-loader/assembly/descriptor/assembly.xml index f67f1e832..5a9cf46f3 100644 --- a/hugegraph-loader/assembly/descriptor/assembly.xml +++ b/hugegraph-loader/assembly/descriptor/assembly.xml @@ -73,6 +73,54 @@ false runtime false + + + ch.qos.reload4j:reload4j + com.esri.geometry:esri-geometry-api + com.fasterxml.jackson.datatype:jackson-datatype-jdk8 + com.fasterxml.jackson.datatype:jackson-datatype-jsr310 + com.fasterxml.jackson.jaxrs:jackson-jaxrs-base + com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider + com.zaxxer:HikariCP + com.zendesk:mysql-binlog-connector-java + io.debezium:debezium-api + io.debezium:debezium-connector-mysql + io.debezium:debezium-core + io.debezium:debezium-ddl-parser + io.debezium:debezium-embedded + jakarta.ws.rs:jakarta.ws.rs-api + javax.activation:activation + javax.ws.rs:javax.ws.rs-api + javax.xml.bind:jaxb-api + net.sourceforge.argparse4j:argparse4j + org.antlr:antlr4-runtime + org.apache.flink:flink-cdc-common + org.apache.flink:flink-cdc-flink2-compat + org.apache.flink:flink-cdc-runtime + org.apache.flink:flink-connector-debezium + org.apache.flink:flink-connector-mysql-cdc + org.apache.flink:flink-shaded-guava + org.apache.kafka:connect-api + org.apache.kafka:connect-file + org.apache.kafka:connect-json + org.apache.kafka:connect-runtime + org.apache.kafka:connect-transforms + org.apache.kafka:kafka-tools + org.apache.maven:maven-artifact + org.bitbucket.b_c:jose4j + org.codehaus.plexus:plexus-utils + org.eclipse.jetty:jetty-continuation + org.eclipse.jetty:jetty-servlets + org.glassfish.hk2.external:aopalliance-repackaged + org.glassfish.hk2.external:jakarta.inject + org.glassfish.hk2:hk2-api + org.glassfish.hk2:hk2-locator + org.glassfish.hk2:hk2-utils + org.glassfish.jersey.containers:jersey-container-servlet + org.glassfish.jersey.inject:jersey-hk2 + org.reflections:reflections + *:*:jar:* diff --git a/hugegraph-loader/assembly/static/bin/hugegraph-flinkcdc-loader.sh b/hugegraph-loader/assembly/static/bin/hugegraph-flinkcdc-loader.sh index 7a6eacc48..ce812fbaf 100755 --- a/hugegraph-loader/assembly/static/bin/hugegraph-flinkcdc-loader.sh +++ b/hugegraph-loader/assembly/static/bin/hugegraph-flinkcdc-loader.sh @@ -15,21 +15,39 @@ # License for the specific language governing permissions and limitations # under the License. # +set -euo pipefail + BIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -APP_DIR=$(dirname "${BIN_DIR}") -LIB_DIR=${APP_DIR}/lib -ASSEMBLY_JAR_NAME=$(find "${LIB_DIR}" -name 'hugegraph-loader*.jar') +APP_DIR=$(dirname "$BIN_DIR") +if [[ -z "${FLINK_HOME:-}" || ! -x "$FLINK_HOME/bin/flink" ]]; then + echo "FLINK_HOME must point to a Flink installation" >&2 + exit 1 +fi +shopt -s nullglob +ASSEMBLY_JARS=("$APP_DIR"/lib/*hugegraph-loader*-shaded.jar) +if [[ ${#ASSEMBLY_JARS[@]} -ne 1 ]]; then + echo "Expected exactly one shaded HugeGraph Loader jar in $APP_DIR/lib" >&2 + exit 1 +fi -# get hugegraph_params and engine_params -source "$BIN_DIR"/get-params.sh -get_params "$@" -echo "engine_params: $ENGINE_PARAMS" -echo "hugegraph_params: $HUGEGRAPH_PARAMS" +# Without a separator all arguments belong to Loader. To set Flink options: +# hugegraph-flinkcdc-loader.sh -p 1 -- --file mapping.json --graph hugegraph +ENGINE_PARAMS=() +HUGEGRAPH_PARAMS=("$@") +for arg in "$@"; do + if [[ "$arg" == "--" ]]; then + HUGEGRAPH_PARAMS=() + while [[ "$1" != "--" ]]; do + ENGINE_PARAMS+=("$1") + shift + done + shift + HUGEGRAPH_PARAMS=("$@") + break + fi +done -CMD=${FLINK_HOME}/bin/flink run \ - "${ENGINE_PARAMS}" \ +# Do not echo arguments: they can contain authentication credentials. +exec "$FLINK_HOME/bin/flink" run ${ENGINE_PARAMS[@]+"${ENGINE_PARAMS[@]}"} \ -c org.apache.hugegraph.loader.flink.HugeGraphFlinkCDCLoader \ - "${ASSEMBLY_JAR_NAME}" "${HUGEGRAPH_PARAMS}" - -echo "${CMD}" -exec "${CMD}" + "${ASSEMBLY_JARS[0]}" ${HUGEGRAPH_PARAMS[@]+"${HUGEGRAPH_PARAMS[@]}"} diff --git a/hugegraph-loader/assembly/static/bin/hugegraph-loader.sh b/hugegraph-loader/assembly/static/bin/hugegraph-loader.sh index dcb0ba0d8..fcd7fe62a 100755 --- a/hugegraph-loader/assembly/static/bin/hugegraph-loader.sh +++ b/hugegraph-loader/assembly/static/bin/hugegraph-loader.sh @@ -69,7 +69,8 @@ CP="$CP":$(find -L ${LIB} -name 'hugegraph*.jar' | sort | tr '\n' ':') # Add the remaining jars in lib. CP="$CP":$(find -L ${LIB} -name '*.jar' \ \! -name 'hugegraph*' \ - \! -name 'log4j-slf4j-impl*.jar' | sort | tr '\n' ':') + \! -name 'log4j-slf4j-impl*.jar' \ + \! -name 'apache-hugegraph-loader-*-shaded.jar' | sort | tr '\n' ':') export LOADER_CLASSPATH="${CLASSPATH:-}:$CP" diff --git a/hugegraph-loader/assembly/travis/flink-cdc/test_verify.py b/hugegraph-loader/assembly/travis/flink-cdc/test_verify.py new file mode 100644 index 000000000..33bd12d78 --- /dev/null +++ b/hugegraph-loader/assembly/travis/flink-cdc/test_verify.py @@ -0,0 +1,154 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gzip +import hashlib +import io +import json +from pathlib import Path +import subprocess +import sys +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch +import xml.etree.ElementTree as ET + +from verify import Acceptance, GRAPHS, normalize, redact, request, set_property, validate_candidate_sdk, validate_jdbc_driver + + +class AcceptanceHelpersTest(unittest.TestCase): + def test_identity_api_probe_uses_owned_graph_port_and_isolated_repository(self): + with tempfile.TemporaryDirectory() as directory: + args = SimpleNamespace(work_root=directory, evidence=directory + "/evidence", + server_sha="server", toolchain_sha="toolchain", maven_repo=directory + "/m2") + acceptance = Acceptance(args) + suite = ET.Element("testsuite", tests="1", failures="0", errors="0", skipped="0") + ET.SubElement(suite, "testcase", name="testIdentityChangesDeletesAndReplay") + with patch.object(acceptance, "run", return_value="probe output") as run, \ + patch("verify.ET.parse", return_value=ET.ElementTree(suite)) as report: + acceptance.verify_identity_api() + command = run.call_args.args[0] + self.assertEqual(command[:3], ["mvn", "-o", "test"]) + self.assertEqual(command[command.index("-pl") + 1], "hugegraph-client,hugegraph-loader") + self.assertIn("-Dtest=HugeGraphIdentityIntegrationTest", command) + self.assertIn("-Dmaven.repo.local=" + str(Path(args.maven_repo).resolve()), command) + self.assertIn("-Dflink.identity.integration=true", command) + self.assertIn("-Dflink.identity.port=38080", command) + self.assertIn("-Dflink.identity.graph=" + GRAPHS[0], command) + self.assertIn(acceptance.prefix + "-identity.xml", str(report.call_args.args[0])) + self.assertEqual(acceptance.report["phases"]["identity_api_probe"]["counts"]["skipped"], 0) + + def test_identity_api_probe_rejects_skipped_missing_or_failed_test(self): + with tempfile.TemporaryDirectory() as directory: + args = SimpleNamespace(work_root=directory, evidence=directory + "/evidence", + server_sha="server", toolchain_sha="toolchain", maven_repo=directory + "/m2") + acceptance = Acceptance(args) + for counts, name in (({"tests": "1", "skipped": "1"}, "testIdentityChangesDeletesAndReplay"), + ({"tests": "0"}, "other"), + ({"tests": "1", "failures": "1"}, "testIdentityChangesDeletesAndReplay"), + ({"tests": "1", "errors": "1"}, "testIdentityChangesDeletesAndReplay"), + ({"tests": "1"}, "other")): + with self.subTest(counts=counts, name=name): + suite = ET.Element("testsuite", **counts) + ET.SubElement(suite, "testcase", name=name) + with patch.object(acceptance, "run", return_value="probe output"), \ + patch("verify.ET.parse", return_value=ET.ElementTree(suite)), \ + self.assertRaisesRegex(AssertionError, "without failures or skips"): + acceptance.verify_identity_api() + self.assertNotIn("identity_api_probe", acceptance.report["phases"]) + + def test_identity_api_failure_stops_before_starting_flink_mysql(self): + with tempfile.TemporaryDirectory() as directory: + archive = Path(directory) / "server.tar.gz" + archive.write_bytes(b"test-only-archive") + manifest = Path(directory) / "candidate-sdk.json" + manifest.write_text(json.dumps({"commit": "server"})) + args = SimpleNamespace(work_root=directory, evidence=directory + "/evidence", + server_sha="server", toolchain_sha="toolchain", sdk_manifest=str(manifest), + server_archive=str(archive), jdbc_driver="driver", + server_archive_sha256=hashlib.sha256(archive.read_bytes()).hexdigest()) + acceptance = Acceptance(args) + with patch("verify.validate_jdbc_driver", return_value={}), patch("verify.socket.socket"), \ + patch.object(acceptance, "start_server") as server, \ + patch.object(acceptance, "verify_identity_api", side_effect=RuntimeError("API probe failed")), \ + patch.object(acceptance, "start_flink_mysql") as engine, \ + self.assertRaisesRegex(RuntimeError, "API probe failed"): + acceptance.verify() + server.assert_called_once_with() + engine.assert_not_called() + + def test_null_removal_is_distinct_from_literal_null_and_stale_value(self): + source = normalize([{"id": 2, "name": "two", "age": 22, "city": None}]) + empty = normalize([{"id": 2, "properties": {"name": "two", "age": 22}}], graph=True) + self.assertEqual(source, empty) + for value in ("null", "old-city"): + stale = normalize([{"id": 2, "properties": {"name": "two", "age": 22, "city": value}}], graph=True) + self.assertNotEqual(source, stale) + + def test_duplicate_ids_are_not_hidden_by_dictionary_conversion(self): + with self.assertRaises(AssertionError): + normalize([{"id": 1}, {"id": 1}]) + + def test_graph_id_type_and_unexpected_properties_fail(self): + with self.assertRaises(AssertionError): + normalize([{"id": "1", "properties": {}}], graph=True) + with self.assertRaises(AssertionError): + normalize([{"id": 1, "properties": {"extra": "unexpected"}}], graph=True) + + def test_decodes_compressed_graph_api_response(self): + response = io.BytesIO(gzip.compress(b'{"vertices": []}')) + response.headers = {"Content-Encoding": "gzip"} + with patch("verify.urllib.request.urlopen", return_value=response): + self.assertEqual(request("http://fixture/vertices"), {"vertices": []}) + + def test_redacts_credential_fields(self): + self.assertNotIn("fake-value", redact('password=fake-value token:fake-value "token":"fake-value"')) + + def test_property_replacement_does_not_create_multiple_active_values(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "server.properties" + path.write_text("#store=example\nstore=old\n") + set_property(path, "store", "new") + set_property(path, "port", "38080") + self.assertEqual(path.read_text().count("store=new"), 1) + self.assertNotIn("store=old", path.read_text()) + self.assertIn("port=38080", path.read_text()) + + def test_rejects_wrong_external_driver_before_runtime_start(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "driver.jar" + path.write_bytes(b"wrong-driver") + with self.assertRaisesRegex(AssertionError, "pinned mysql:mysql-connector-java:8.0.28"): + validate_jdbc_driver(path) + + def test_rejects_sdk_manifest_from_another_server_source(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "candidate-sdk.json" + selected = "a" * 40 + path.write_text(json.dumps({"commit": "b" * 40})) + with self.assertRaisesRegex(AssertionError, "selected server source"): + validate_candidate_sdk(path, selected) + path.write_text(json.dumps({"commit": selected})) + self.assertEqual(validate_candidate_sdk(path, selected)["commit"], selected) + + def test_default_mode_does_not_start_or_require_docker(self): + script = Path(__file__).with_name("verify.py") + output = subprocess.check_output([sys.executable, str(script)], env={"PATH": ""}, text=True) + self.assertEqual(json.loads(output)["mode"], "plan") + + +if __name__ == "__main__": + unittest.main() diff --git a/hugegraph-loader/assembly/travis/flink-cdc/verify.py b/hugegraph-loader/assembly/travis/flink-cdc/verify.py new file mode 100644 index 000000000..9e6c27c2f --- /dev/null +++ b/hugegraph-loader/assembly/travis/flink-cdc/verify.py @@ -0,0 +1,475 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Linux-only, test-owned Flink/MySQL/server acceptance. Default is a no-I/O plan.""" + +import argparse +import gzip +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import time +import urllib.request +from urllib.parse import urlsplit +import uuid +import xml.etree.ElementTree as ET + +FLINK_IMAGE = "flink:2.2.1-java17" +MYSQL_IMAGE = "mysql:8.0.44" +JDBC_COORDINATE = "mysql:mysql-connector-java:8.0.28" +JDBC_SHA256 = "a00ccdf537ff50e50067b989108c2235197ffb65e197149bbb669db843cd1c3e" +GRAPHS = ("flink_initial", "flink_rebuild") +SERVER_URL = "http://127.0.0.1:38080" +FLINK_URL = "http://127.0.0.1:18081" + + +def validate_jdbc_driver(path): + digest = hashlib.sha256(Path(path).read_bytes()).hexdigest() + if digest != JDBC_SHA256: + raise AssertionError("External test JDBC driver does not match pinned " + JDBC_COORDINATE) + return {"coordinate": JDBC_COORDINATE, "sha256": digest, "distribution": "external-test-runtime-only"} + + +def validate_candidate_sdk(path, source_commit): + manifest = json.loads(Path(path).read_text()) + if manifest.get("commit") != source_commit: + raise AssertionError("Candidate SDK manifest does not match the selected server source") + return manifest + + +def redact(text): + return re.sub(r"(?i)((?:password|token|authorization)[\"']?\s*[=:]\s*)[^\s,;]+", + r"\1[REDACTED]", text) + + +def normalize(rows, graph=False): + result = {} + for row in rows: + if graph and type(row["id"]) is not int: + raise AssertionError("Expected numeric graph ID") + key = str(row["id"]) + if key in result: + raise AssertionError("Duplicate graph/source ID: " + key) + props = row["properties"] if graph else row + if graph and set(props) - {"name", "age", "city"}: + raise AssertionError("Unexpected graph properties") + result[key] = {name: props.get(name) for name in ("name", "age", "city")} + return result + + +def request(url, body=None): + data = None if body is None else json.dumps(body).encode() + req = urllib.request.Request(url, data=data, headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=10) as response: + body = response.read() + if response.headers.get("Content-Encoding", "").lower() == "gzip": + body = gzip.decompress(body) + return json.loads(body) + + +def wait_for(description, probe, seconds=180): + deadline = time.monotonic() + seconds + last_error = "condition not satisfied" + while time.monotonic() < deadline: + try: + value = probe() + if value: + return value + except (OSError, ValueError, RuntimeError) as error: + last_error = str(error) + time.sleep(1) + raise TimeoutError(description + ": " + redact(last_error)) + + +def set_property(path, key, value): + text = path.read_text() + pattern = re.compile(r"^\s*" + re.escape(key) + r"\s*=.*$", re.MULTILINE) + entry = key + "=" + value + path.write_text(pattern.sub(entry, text) if pattern.search(text) else text + "\n" + entry + "\n") + + +class Acceptance: + def __init__(self, args): + self.args = args + self.work = Path(tempfile.mkdtemp(prefix="hg-flink-", dir=args.work_root)) + self.evidence = Path(args.evidence).resolve() + self.evidence.mkdir(parents=True, exist_ok=True) + self.prefix = "hg-flink-" + uuid.uuid4().hex[:10] + self.containers = [] + self.server_process = None + self.server_log = None + self.report = {"status": "running", "phases": {}, "server_sha": args.server_sha, + "toolchain_sha": args.toolchain_sha, "work": str(self.work)} + + def run(self, command, *, cwd=None, env=None, timeout=300, input_text=None): + result = subprocess.run(command, cwd=cwd, env=env, input=input_text, + text=True, capture_output=True, timeout=timeout) + if result.returncode: + raise RuntimeError(f"{command[0]} exited {result.returncode}: " + + redact((result.stdout + result.stderr)[-6000:])) + return result.stdout + + def save(self, name, value): + (self.evidence / (name + ".json")).write_text(json.dumps(value, indent=2) + "\n") + + def phase(self, name, value): + self.report["phases"][name] = value + self.save("result", self.report) + + def start_server(self): + extracted = self.work / "server" + extracted.mkdir() + self.run(["tar", "xzf", str(Path(self.args.server_archive).resolve()), "-C", str(extracted)]) + roots = list(extracted.glob("apache-hugegraph-*/")) + if len(roots) != 1: + raise AssertionError("Expected one server distribution") + self.server = roots[0] + conf = self.server / "conf" + template = (conf / "graphs/hugegraph.properties").read_text() + # This is a newly extracted distribution inside this run's private directory. + for old in (conf / "graphs").glob("*.properties"): + old.unlink() + for graph in GRAPHS: + props = conf / "graphs" / (graph + ".properties") + props.write_text(template) + set_property(props, "store", graph) + set_property(props, "backend", "rocksdb") + set_property(props, "rocksdb.data_path", str(self.work / (graph + "-data"))) + set_property(props, "rocksdb.wal_path", str(self.work / (graph + "-wal"))) + set_property(conf / "rest-server.properties", "restserver.url", SERVER_URL) + set_property(conf / "rest-server.properties", "gremlinserver.url", "http://127.0.0.1:18182") + yaml = conf / "gremlin-server.yaml" + yaml.write_text(re.sub(r"^#?port:.*$", "port: 18182", yaml.read_text(), flags=re.MULTILINE)) + env = dict(os.environ, JAVA_TOOL_OPTIONS="-Xmx512m -XX:ActiveProcessorCount=2") + self.run(["bash", "bin/init-store.sh"], cwd=self.server, env=env) + self.server_log = (self.work / "server-console.log").open("w") + self.server_process = subprocess.Popen( + ["bash", "bin/start-hugegraph.sh", "-d", "false", "-j", + "-Xms256m -Xmx512m -XX:ActiveProcessorCount=2", "-t", "120"], + cwd=self.server, env=env, stdout=self.server_log, stderr=subprocess.STDOUT, + start_new_session=True) + wait_for("server graphs", lambda: request(self.graph_url(GRAPHS[0]) + "/schema/propertykeys")) + for graph in GRAPHS: + url = self.graph_url(graph) + "/schema/" + for name, kind in (("name", "TEXT"), ("age", "INT"), ("city", "TEXT")): + request(url + "propertykeys", {"name": name, "data_type": kind, "cardinality": "SINGLE"}) + request(url + "vertexlabels", {"name": "cdc_person", "id_strategy": "CUSTOMIZE_NUMBER", + "properties": ["name", "age", "city"], "nullable_keys": ["city"]}) + self.phase("server_ready", {"graphs": GRAPHS, "java": self.java_version()}) + + @staticmethod + def java_version(): + result = subprocess.run([str(Path(os.environ["JAVA_HOME"]) / "bin/java"), "-version"], + text=True, capture_output=True, check=True) + if not re.search(r'version "17[.\"]', result.stderr): + raise AssertionError("Server JVM must be Java 17") + return result.stderr.strip() + + @staticmethod + def graph_url(graph): + return SERVER_URL + "/graphspaces/DEFAULT/graphs/" + graph + + def verify_identity_api(self): + # Use only the freshly extracted Server owned by this acceptance run. + root = Path(__file__).resolve().parents[4] + suffix = self.prefix + "-identity" + port = urlsplit(SERVER_URL).port + test = "org.apache.hugegraph.loader.flink.HugeGraphIdentityIntegrationTest" + output = self.run(["mvn", "-o", "test", "-pl", "hugegraph-client,hugegraph-loader", "-am", "-ntp", + "-Dmaven.repo.local=" + str(Path(self.args.maven_repo).resolve()), + "-Dtest=HugeGraphIdentityIntegrationTest", "-Dsurefire.failIfNoSpecifiedTests=false", + "-DforkCount=0", "-Dflink.identity.integration=true", + "-Dflink.identity.port=" + str(port), "-Dflink.identity.graph=" + GRAPHS[0], + "-Dsurefire.reportNameSuffix=" + suffix], cwd=root, timeout=300) + (self.evidence / "identity-api-maven.log").write_text(redact(output[-60000:])) + report = root / "hugegraph-loader/target/surefire-reports" / ("TEST-" + test + "-" + suffix + ".xml") + suite = ET.parse(report).getroot() + counts = {name: int(suite.get(name, "0")) for name in ("tests", "failures", "errors", "skipped")} + if counts["tests"] != 1 or any(counts[name] for name in ("failures", "errors", "skipped")) or \ + not any(case.get("name") == "testIdentityChangesDeletesAndReplay" for case in suite.findall("testcase")): + raise AssertionError("Identity API probe must execute its test without failures or skips") + self.phase("identity_api_probe", {"test": test, "counts": counts, "port": port, "graph": GRAPHS[0], + "report_suffix": suffix, "scope": "SourceRecord-to-Graph-API; not Flink edge E2E"}) + + def container(self, suffix, image, arguments, environment, mounts=(), memory="1200m"): + name = self.prefix + "-" + suffix + command = ["docker", "run", "-d", "--name", name, "--network", "host", "--memory", memory, + "--label", "hugegraph.flink.acceptance=" + self.prefix] + for key, value in environment.items(): + command.extend(["-e", key + "=" + value]) + for mount in mounts: + command.extend(["-v", mount]) + # Record first so a partially successful Docker invocation is still cleaned up. + self.containers.append(name) + self.run(command + [image] + arguments) + return name + + def mysql(self, sql): + return self.run(["docker", "exec", "-i", self.mysql_name, "mysql", "-uroot", "--protocol=TCP", + "-h127.0.0.1", "-P13306", "--raw", "--batch", "--skip-column-names"], + input_text=sql) + + def start_flink_mysql(self): + for image in (FLINK_IMAGE, MYSQL_IMAGE): + self.run(["docker", "pull", image], timeout=600) + self.save("images", {image: json.loads(self.run(["docker", "image", "inspect", image, + "--format", "{{json .RepoDigests}} "])) for image in (FLINK_IMAGE, MYSQL_IMAGE)}) + self.mysql_name = self.container("mysql", MYSQL_IMAGE, + ["--port=13306", "--bind-address=127.0.0.1", "--server-id=1", "--log-bin=mysql-bin", + "--binlog-format=ROW", "--binlog-row-image=FULL"], {"MYSQL_ALLOW_EMPTY_PASSWORD": "yes"}, + memory="768m") + wait_for("MySQL ready", lambda: self.mysql("SELECT 1;")) + settings = self.mysql("SELECT @@version, @@binlog_format, @@binlog_row_image, @@log_bin;").strip() + self.save("mysql-binlog", {"settings": settings}) + if settings.split("\t")[-3:] != ["ROW", "FULL", "1"]: + raise AssertionError("MySQL must have ROW/FULL binary logging enabled") + self.mysql("CREATE DATABASE flink_fixture; CREATE TABLE flink_fixture.person " + "(id INT PRIMARY KEY, name VARCHAR(50) NOT NULL, age INT NOT NULL, city VARCHAR(50)); " + "INSERT INTO flink_fixture.person VALUES (1,'one',21,'a'),(2,'two',22,'b'),(3,'three',23,NULL);") + job = self.work / "job" + (job / "lib").mkdir(parents=True) + (job / "bin").mkdir() + jar = Path(self.args.job_jar).resolve() + destination = job / "lib" / jar.name + try: + os.link(jar, destination) + except OSError: + shutil.copyfile(jar, destination) + with jar.open("rb") as stream: + self.report["job_sha256"] = hashlib.file_digest(stream, "sha256").hexdigest() + shutil.copyfile(Path(__file__).parents[2] / "static/bin/hugegraph-flinkcdc-loader.sh", + job / "bin/hugegraph-flinkcdc-loader.sh") + mapping = {"version": "2.0", "structs": [{"id": "mysql", "input": { + "type": "jdbc", "vendor": "mysql", "url": "jdbc:mysql://127.0.0.1:13306", + "database": "flink_fixture", "table": "person", "username": "root", "password": "", + "header": ["id", "name", "age", "city"]}, "vertices": [{"label": "cdc_person", "id": "id", + "update_strategies": {"name": "OVERRIDE", "age": "OVERRIDE", "city": "OVERRIDE"}}]}]} + (job / "mapping.json").write_text(json.dumps(mapping)) + checkpoints = self.work / "checkpoints" + checkpoints.mkdir(mode=0o777) + checkpoints.chmod(0o777) + config = "\n".join(["jobmanager.rpc.address: 127.0.0.1", "jobmanager.rpc.port: 16123", + "rest.address: 127.0.0.1", "rest.bind-address: 127.0.0.1", "rest.port: 18081", + "blob.server.port: 16124", "taskmanager.bind-host: 127.0.0.1", "taskmanager.host: 127.0.0.1", + "taskmanager.rpc.port: 16122", "taskmanager.numberOfTaskSlots: 1", "parallelism.default: 1", + "jobmanager.memory.process.size: 1024m", "taskmanager.memory.process.size: 1536m", + "state.checkpoints.dir: file:///checkpoints", "state.savepoints.dir: file:///checkpoints", + "execution.checkpointing.externalized-checkpoint-retention: RETAIN_ON_CANCELLATION", + "restart-strategy.type: fixed-delay", "restart-strategy.fixed-delay.attempts: 5", + "restart-strategy.fixed-delay.delay: 1 s"]) + environment = {"FLINK_PROPERTIES": config, "JAVA_TOOL_OPTIONS": "-XX:ActiveProcessorCount=2"} + shared = [str(checkpoints) + ":/checkpoints", + str(Path(self.args.jdbc_driver).resolve()) + ":/opt/flink/lib/mysql-connector-java-8.0.28.jar:ro"] + self.jm = self.container("jm", FLINK_IMAGE, ["jobmanager"], environment, + shared + [str(job) + ":/job:ro"]) + # Worker deliberately has no mapping-file or Loader JAR mount: Flink must ship the job. + self.tm = self.container("tm", FLINK_IMAGE, ["taskmanager"], environment, shared, memory="1800m") + wait_for("TaskManager registered", lambda: request(FLINK_URL + "/taskmanagers")["taskmanagers"]) + for name in (self.jm, self.tm): + result = self.run(["docker", "exec", name, "sh", "-c", "java -version 2>&1"]) + if not re.search(r'version "17[.\"]', result): + raise AssertionError("Flink JVM is not Java 17") + self.phase(name.rsplit("-", 1)[-1] + "_java", result.strip()) + + def submit(self, graph, savepoint=None): + options = ["-d", "-p", "1"] + if savepoint: + options += ["-s", savepoint] + output = self.run(["docker", "exec", self.jm, "bash", "/job/bin/hugegraph-flinkcdc-loader.sh"] + + options + ["--", "--file", "/job/mapping.json", "--graph", graph, "--host", "127.0.0.1", + "--port", "38080", "--cdc-sink-parallelism", "1", "--cdc-flush-interval", "0"]) + match = re.search(r"JobID\s+([0-9a-f]{32})", output) + if not match: + raise AssertionError("Flink did not return a submitted JobID: " + redact(output)) + job = match.group(1) + def running(): + state = request(FLINK_URL + "/jobs/" + job)["state"] + if state in ("FAILED", "CANCELED", "FINISHED"): + raise AssertionError("CDC job unexpectedly " + state) + return state == "RUNNING" + wait_for("job running", running) + self.save("job-" + job, request(FLINK_URL + "/jobs/" + job)) + return job + + def source_rows(self): + lines = self.mysql("SELECT JSON_OBJECT('id',id,'name',name,'age',age,'city',city) " + "FROM flink_fixture.person ORDER BY id;").splitlines() + return normalize([json.loads(line) for line in lines]) + + def graph_rows(self, graph): + data = request(self.graph_url(graph) + "/graph/vertices?label=cdc_person&limit=100") + return normalize(data["vertices"], graph=True) + + def compare(self, phase, graph, job): + expected = self.source_rows() + observed = {} + + def probe(): + nonlocal observed + state = request(FLINK_URL + "/jobs/" + job)["state"] + if state in ("FAILED", "CANCELED", "FINISHED"): + raise AssertionError("CDC job unexpectedly " + state) + observed = self.graph_rows(graph) + return observed == expected + + try: + wait_for(phase, probe) + finally: + self.save(phase + "-rows", {"expected": expected, "actual": observed, "job": job}) + self.phase(phase, {"job": job, "rows": len(expected)}) + + def checkpoint(self, job, after=-1): + def probe(): + state = request(FLINK_URL + "/jobs/" + job + "/checkpoints") + completed = state.get("latest", {}).get("completed") + return completed if completed and completed["id"] > after else None + return wait_for("completed checkpoint", probe) + + def cancel(self, job): + self.run(["docker", "exec", self.jm, "/opt/flink/bin/flink", "cancel", job]) + wait_for("job canceled", lambda: request(FLINK_URL + "/jobs/" + job)["state"] == "CANCELED") + + def verify(self): + self.report["external_jdbc_driver"] = validate_jdbc_driver(self.args.jdbc_driver) + manifest = validate_candidate_sdk(self.args.sdk_manifest, self.args.server_sha) + self.save("candidate-sdk", manifest) + with Path(self.args.sdk_manifest).open("rb") as stream: + self.report["sdk_manifest_sha256"] = hashlib.file_digest(stream, "sha256").hexdigest() + with Path(self.args.server_archive).open("rb") as stream: + self.report["server_archive_sha256"] = hashlib.file_digest(stream, "sha256").hexdigest() + if self.report["server_archive_sha256"] != self.args.server_archive_sha256: + raise AssertionError("Server archive does not match the same-source SDK action output") + for port in (38080, 18182, 13306, 18081, 16123, 16124, 16122): + with socket.socket() as probe: + probe.bind(("127.0.0.1", port)) + self.start_server() + self.verify_identity_api() + self.start_flink_mysql() + job = self.submit(GRAPHS[0]) + self.compare("snapshot", GRAPHS[0], job) + checkpoint = self.checkpoint(job) + savepoint_output = self.run(["docker", "exec", self.jm, "/opt/flink/bin/flink", "savepoint", job, + "file:///checkpoints"]) + match = re.search(r"(file:/+checkpoints/savepoint-[^\s]+)", savepoint_output) + if not match: + raise AssertionError("Flink did not return a savepoint path") + savepoint = match.group(1).rstrip(".") + self.phase("snapshot_checkpoint", {"checkpoint": checkpoint, "savepoint": savepoint}) + self.mysql("INSERT INTO flink_fixture.person VALUES(4,'four',24,'d'); " + "UPDATE flink_fixture.person SET name='updated',age=42,city=NULL WHERE id=2; " + "DELETE FROM flink_fixture.person WHERE id=1;") + self.compare("changes_and_null", GRAPHS[0], job) + previous = request(FLINK_URL + "/taskmanagers")["taskmanagers"][0]["id"] + self.run(["docker", "restart", self.tm]) + self.mysql("INSERT INTO flink_fixture.person VALUES(5,'five',25,'e'); " + "UPDATE flink_fixture.person SET city='after-restart' WHERE id=3;") + registered = wait_for("replacement TaskManager", lambda: [tm for tm in + request(FLINK_URL + "/taskmanagers")["taskmanagers"] if tm["id"] != previous]) + self.compare("taskmanager_restart", GRAPHS[0], job) + self.phase("restart_checkpoint", {"previous_taskmanager": previous, "replacement": registered, + "checkpoint": self.checkpoint(job, checkpoint["id"])}) + # Debezium emits DELETE + CREATE for a changed database primary key. + self.mysql("UPDATE flink_fixture.person SET id=30 WHERE id=3;") + self.compare("primary_key_change", GRAPHS[0], job) + # Force deterministic replay, including an already-applied delete, from the older savepoint. + self.cancel(job) + job = self.submit(GRAPHS[0], savepoint) + # A new binlog marker prevents already-correct graph data from passing before replay catches up. + self.mysql("UPDATE flink_fixture.person SET name='after-replay' WHERE id=5;") + self.compare("savepoint_replay", GRAPHS[0], job) + self.phase("replay_checkpoint", self.checkpoint(job)) + self.cancel(job) + self.mysql("DELETE FROM flink_fixture.person WHERE id=2; " + "INSERT INTO flink_fixture.person VALUES(6,'six',26,NULL); " + "UPDATE flink_fixture.person SET name='during-downtime' WHERE id=4;") + if self.graph_rows(GRAPHS[1]): + raise AssertionError("Rebuild target must start empty") + job = self.submit(GRAPHS[1]) + self.compare("fresh_target_rebuild", GRAPHS[1], job) + self.phase("rebuild_checkpoint", self.checkpoint(job)) + self.cancel(job) + self.report["status"] = "passed" + + def cleanup(self): + for name in self.containers: + owner = subprocess.run(["docker", "inspect", "--format", + '{{index .Config.Labels "hugegraph.flink.acceptance"}}', name], text=True, capture_output=True) + if owner.returncode or owner.stdout.strip() != self.prefix: + continue + logs = subprocess.run(["docker", "logs", "--tail", "500", name], text=True, capture_output=True) + (self.evidence / (name.rsplit("-", 1)[-1] + ".log")).write_text(redact(logs.stdout + logs.stderr)) + subprocess.run(["docker", "rm", "-f", name], capture_output=True) + if self.server_process and self.server_process.poll() is None: + os.killpg(self.server_process.pid, signal.SIGTERM) + try: + self.server_process.wait(timeout=15) + except subprocess.TimeoutExpired: + os.killpg(self.server_process.pid, signal.SIGKILL) + if self.server_log: + self.server_log.close() + (self.evidence / "server.log").write_text(redact((self.work / "server-console.log").read_text()[-60000:])) + self.save("result", self.report) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run", action="store_true", help="start test-owned Linux runtime resources") + parser.add_argument("--server-archive") + parser.add_argument("--server-archive-sha256", help="archive hash from the same-source SDK action") + parser.add_argument("--sdk-manifest", help="manifest from the same-source SDK action") + parser.add_argument("--maven-repo", help="isolated repository from the same-source SDK action, for the API probe") + parser.add_argument("--job-jar") + parser.add_argument("--jdbc-driver", help="external mysql-connector-java 8.0.28 JAR; never bundled in the job") + parser.add_argument("--server-sha", default="unknown") + parser.add_argument("--toolchain-sha", default="unknown") + parser.add_argument("--work-root", default=tempfile.gettempdir()) + parser.add_argument("--evidence", default="flink-cdc-evidence") + args = parser.parse_args() + if not args.run: + print(json.dumps({"mode": "plan", "images": [FLINK_IMAGE, MYSQL_IMAGE], "graphs": GRAPHS, + "phases": ["identity_api_probe", "snapshot", "changes_and_null", "taskmanager_restart", "primary_key_change", + "savepoint_replay", "fresh_target_rebuild"]}, indent=2)) + return + if sys.platform != "linux" or not all((args.server_archive, args.server_archive_sha256, + args.sdk_manifest, args.maven_repo, args.job_jar, args.jdbc_driver)): + parser.error("--run requires Linux, --server-archive, --server-archive-sha256, " + "--sdk-manifest, --maven-repo, --job-jar and --jdbc-driver") + if not Path(args.maven_repo).is_dir(): + parser.error("--maven-repo must be an existing isolated repository") + if not re.fullmatch(r"[0-9a-f]{64}", args.server_archive_sha256): + parser.error("--run requires a full server archive SHA-256") + if not all(re.fullmatch(r"[0-9a-fA-F]{40}", sha) for sha in (args.server_sha, args.toolchain_sha)): + parser.error("--run requires full server and Toolchain commit SHAs") + acceptance = Acceptance(args) + try: + acceptance.verify() + except Exception as error: + acceptance.report.update(status="failed", error=redact(str(error))) + raise + finally: + acceptance.cleanup() + + +if __name__ == "__main__": + main() diff --git a/hugegraph-loader/docs/flink-cdc-java17.md b/hugegraph-loader/docs/flink-cdc-java17.md new file mode 100644 index 000000000..1b92b3b2e --- /dev/null +++ b/hugegraph-loader/docs/flink-cdc-java17.md @@ -0,0 +1,101 @@ +# Flink CDC on Java 17 + +This integration targets Flink 2.2.1 and MySQL CDC 3.6.0-2.2 on Java 17. CI first checks rejection of vertex and edge identity changes, deletion and replay through a Graph API probe against its test-owned, same-source candidate Server. It then checks a bounded vertex job against real MySQL and Flink, covering snapshots, insert/update/NULL/delete, primary-key changes, TaskManager restart, savepoint replay and fresh-target rebuild. The API probe does not verify edge delivery through Flink/MySQL; broader mappings and edge engine coverage require separate validation. These bounded checks do not establish supported production deployment. + +## Building + +The root POM pins the Flink and CDC versions. Compile and test the adaptation with: + +```bash +mvn test -pl hugegraph-loader \ + -Dtest=FlinkSinkSerializationTest,HugeGraphSinkTest,HugeGraphDeserializationTest +node --test hugegraph-loader/src/test/scripts/flink-launcher.test.cjs +``` + +Build and install the matching Java 17 Client and its server-side dependencies first. The Flink runtime remains provided by the cluster. The CDC connector is a runtime part of the job and is included in the Loader shading inputs. CI validates the packaged vertex job with the external JDBC driver and matching Client in its pinned Flink 2.2.1 Java 17 image. CDC's Flink 2 compatibility jar includes duplicate Flink API classes. The POM places the provided Flink core before CDC so compilation and tests use the public 2.2 Sink interface. The packaged class-loading probe checks that the CDC coordinator and Guava 31 load from the job JAR, while Guava 33 and Flink's `Source` API load from the cluster's `flink-dist-2.2.1.jar`. Clusters with different dependencies or class-loading settings require separate validation; passing the compile-time contract test alone is insufficient. + +## Launch arguments + +Set `FLINK_HOME` to the selected Flink installation. The submission JVM, JobManager and TaskManager must all run Java 17. Preserve the distribution's `env.java.opts.all` module-access options; do not replace them with a shorter list. The launcher selects exactly one `*hugegraph-loader*-shaded.jar` in `lib`. Spark retains the same shared job artifact path. The ordinary Loader launcher excludes this shaded job from its classpath, and the assembly omits CDC-only loose dependencies. Kafka clients and their compression dependency remain available to the ordinary Kafka reader; common dependency versions are retained for the shared shaded job. + +CDC rejects `--dry-run true` and `--use-prefilter true` before creating a sink or opening a client. It does not implement preview writes or ingestion deduplication. UPDATE requires explicit non-empty mapping `update_strategies` and an unchanged graph identity. Identity-changing UPDATE events fail before any mapping is buffered or written; remote failures in supported writes still require replay/recovery. + +Without a separator, all arguments are Loader options: + +```bash +bin/hugegraph-flinkcdc-loader.sh --file mapping.json --graph hugegraph +``` + +Place Flink options before `--` and Loader options after it. Paths with spaces remain individual arguments; the launcher does not print argument values. + +```bash +bin/hugegraph-flinkcdc-loader.sh -p 1 -- \ + --file "my mappings/mysql.json" --graph hugegraph --cdc-sink-parallelism 1 +``` + +Each mapping uses one source reader and one sink writer to preserve mutation order. `--cdc-sink-parallelism` defaults to 1 and other values are rejected before building the pipeline. Both operators explicitly use parallelism 1, including when Flink's global parallelism is unset or `-p 2` is supplied before `--`. This serial topology does not support parallel CDC writers. After `--`, Loader's `-p` is the HugeGraph server port, not Flink parallelism. The launcher and configuration serialization can transport other numeric values; the CDC entry point enforces the supported topology when it receives them. Both Flink submission and direct JVM invocation report invalid sink settings as failures. + +The submitting process validates the mapping file and Loader options. Sink configuration is serialized to workers without requiring that file to exist on each worker. The sink uses the public Sink V2 writer lifecycle. Checkpoint flush delegates to the existing graph output logic; this does not add transactional graph writes or an exactly-once guarantee. + +## Remaining runtime coverage + +Edge CDC, broader mapping configurations and custom cluster environments still need runtime acceptance beyond the bounded vertex fixture. For those cases, compare complete IDs and property values through snapshot, mutations, worker recovery and replay against real MySQL and HugeGraph instances. + +A fresh snapshot into an existing graph does not by itself remove graph records deleted during downtime: controlled rebuild needs an explicit fresh-target or reconciliation procedure. Do not delete user graph data as an implicit migration step. + + +## Reproducible Linux acceptance + +The `flink-cdc-java17-ci` workflow uses the shared same-source SDK action aligned with ASF Server master. It installs the matching Common/Core/PD/Store SDK and Server distribution dependency closure in a fresh Maven repository, then packages the Loader job. Its runtime stage is separate from compilation and unit-test success. + +To inspect the test plan without touching Docker or starting services: + +```bash +python3 hugegraph-loader/assembly/travis/flink-cdc/verify.py +``` + +Actual execution requires an isolated Linux host, Docker, Python 3.11+, Maven and Java 17. Provide the same-source server archive, its populated isolated Maven repository and packaged shaded Loader JAR. The API probe uses offline Maven against that repository: + +```bash +python3 hugegraph-loader/assembly/travis/flink-cdc/verify.py --run \ + --server-archive /path/to/apache-hugegraph.tar.gz \ + --server-archive-sha256 \ + --sdk-manifest /path/to/candidate-sdk-manifest.json \ + --maven-repo /path/to/candidate-m2 \ + --job-jar /path/to/hugegraph-loader-shaded.jar \ + --jdbc-driver /path/to/mysql-connector-java-8.0.28.jar \ + --server-sha --toolchain-sha \ + --evidence /path/to/test-evidence +``` + +The harness reserves loopback ports 38080/18182 (Server), 13306 (MySQL), and 18081/16123/16124/16122 (Flink), refusing occupied ports. It creates a new Server directory and two test graphs, plus uniquely labelled containers. MySQL uses an empty test-only root password, binds to loopback, and never points at an existing database. Cleanup removes only the containers carrying this run's ownership label and stops its own server process group. Do not use production data or hosts. + +JobManager and TaskManager run in separate Java 17 containers. Only JobManager mounts the mapping file and packaged job; TaskManager must receive the actual job through Flink. The test compares numeric vertex IDs and all mapped properties against MySQL after snapshot, insert/update/NULL/delete, SQL primary-key changes, TaskManager restart, forced replay from an earlier savepoint, and a fresh-target rebuild after downtime. A new binlog marker after restore prevents pre-existing correct graph data from passing before replay has caught up. Successful checkpoints are required. + +The harness rejects a manifest from another source commit or an archive differing from the SDK action's SHA-256 before starting runtime resources. Evidence includes the SDK manifest, source commits, archive/job hashes, image digests, JVM versions, checkpoint/savepoint metadata, row comparisons and sanitized logs. A failed or unexecuted runtime stage is not compatibility evidence. The current acceptance fixture covers vertex CDC; edge CDC and broader mapping configurations still need separate runtime coverage before declaring the entire mode supported. + +## Identity and delete behavior + +UPDATE events retain both row images. Configure MySQL with `binlog_row_image=FULL`; a missing before image fails explicitly. The sink derives PRIMARY_KEY vertex IDs through the same Loader builder path used for edge endpoints. It resolves edge IDs through the public adjacency API using source, label and sort keys, then matches the target. Queries preserve literal sort-key strings and paginate when needed. An already absent element is accepted when a delete is replayed. + +UPDATE supports only unchanged vertex IDs, edge endpoints and sort keys, including identities added or removed by unfolded mappings. The sink checks all non-skipped mappings before any write and rejects changes with an explicit error, preserving the old elements and incident edges. Atomic identity migration is tracked in [#793](https://github.com/apache/hugegraph-toolchain/issues/793). Separate DELETE/CREATE events retain their existing non-transactional semantics; this guard does not make Debezium database primary-key changes atomic. Source-to-graph identity mappings must be unambiguous; independent rows must not compete for one graph identity. + +CDC converts non-null source fields to strings before Loader mapping. Configure `null_values` with string markers such as `"0"` or `""`; a numeric marker such as `0` does not match the source string `"0"`. Nullable property updates follow the Loader mapping's selected fields, ignored fields and null markers. + +Run the opt-in REST integration probe against a test-only Server: + +```bash +mvn test -pl hugegraph-client,hugegraph-loader -am \ + -Dtest=HugeGraphIdentityIntegrationTest -Dsurefire.failIfNoSpecifiedTests=false \ + -Dflink.identity.integration=true -Dflink.identity.port=8080 -Dflink.identity.graph=hugegraph +``` + +The probe uses UUID-prefixed schemas in the selected `DEFAULT` graph (default `hugegraph`) and removes only those fixtures. CI selects `flink_initial` on its owned Server at port 38080 and requires a fresh, uniquely suffixed Surefire report with one passing test and zero skips before starting Flink/MySQL. A probe failure fails the acceptance run. It passes real Debezium `SourceRecord` objects through the deserializer and sink. It checks rejection of CUSTOMIZE_NUMBER and PRIMARY_KEY vertex identity changes, preservation of incident edges, UUID vertex delete, rejection of edge endpoint/sort-key changes, same-identity updates, duplicate replay and deletion, and deletion beyond a 100-edge query page, including a literal `P.gt(1)` sort key. This verifies the Graph API contract, not delivery through a running Flink/MySQL cluster. + +The candidate Server's existing edge-create API still rejects UUID endpoints when `check_vertex=true`: JSON UUID IDs arrive as strings and are looked up without converting the endpoint schema type. UUID edge ingestion is therefore not verified by this probe; this separate Server API limitation must be resolved before claiming that mapping supported. + +## External driver and bundled component licenses + +MySQL Connector/J remains an external, provided dependency. Install a compatible driver in the Flink cluster's `lib` directory on both JobManager and TaskManager; it is not included in the Loader shaded JAR or release archive. The test harness uses `mysql:mysql-connector-java:8.0.28` only as an external test-runtime file, checks its pinned SHA-256 before starting services, and mounts it read-only into both containers. CI downloads this file separately and does not upload it with the released job or evidence. + +The Loader's Flink CDC dependency aggregate includes [Activation 1.1.1](https://javaee.github.io/activation/) under CDDL 1.0; [HK2 2.6.1](https://eclipse-ee4j.github.io/glassfish-hk2/) (including its repackaged AOP Alliance and Jakarta Inject components), [JAX-RS APIs 2.1.1/2.1.6](https://github.com/jakartaee/rest), and [Jersey Servlet 2.34/3.1.11 and HK2 integration 2.34](https://eclipse-ee4j.github.io/jersey/) under their EPL 2.0 license option. See the distribution `LICENSE` and `NOTICE` for the full terms and additional embedded-component notices. diff --git a/hugegraph-loader/pom.xml b/hugegraph-loader/pom.xml index ed2d7183c..50d4ec1ca 100644 --- a/hugegraph-loader/pom.xml +++ b/hugegraph-loader/pom.xml @@ -118,45 +118,41 @@ - - com.ververica - flink-connector-mysql-cdc - ${flinkcdc.version} - ${flink.scope} - - - org.apache.flink - flink-java - ${flink.version} - ${flink.scope} - + org.apache.flink - flink-streaming-java_${scala.version} + flink-core ${flink.version} ${flink.scope} org.apache.flink - flink-table-api-java-bridge_${scala.version} - ${flink.version} - ${flink.scope} + flink-connector-mysql-cdc + ${flinkcdc.version} + + + + + mysql + mysql-connector-java + + org.apache.flink - flink-table-planner-blink_${scala.version} + flink-streaming-java ${flink.version} ${flink.scope} org.apache.flink - flink-clients_${scala.version} + flink-clients ${flink.version} ${flink.scope} org.apache.flink - flink-runtime-web_${scala.version} + flink-runtime-web ${flink.version} ${flink.scope} @@ -672,7 +668,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.4.1 + 3.6.1 package @@ -682,6 +678,10 @@ false ${final.name}-shaded + + + org.apache.flink:flink-shaded-guava:31.1-jre-17.0 + com.google.common.base diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/EdgeBuilder.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/EdgeBuilder.java index 950100187..c0e64c59e 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/EdgeBuilder.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/EdgeBuilder.java @@ -118,6 +118,16 @@ public List build(String[] names, Object[] values) { return edges; } + public Map identityProperties(Edge edge) { + Map properties = new HashMap<>(); + for (String key : this.edgeLabel.sortKeys()) { + Object value = edge.property(key); + E.checkArgumentNotNull(value, "The edge identity requires sort key '%s'", key); + properties.put(key, value); + } + return properties; + } + private EdgeKVPairs newEdgeKVPairs() { EdgeKVPairs kvPairs = new EdgeKVPairs(); kvPairs.source = this.newKVPairs(this.sourceLabel, diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/VertexBuilder.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/VertexBuilder.java index 8d006f368..277a66acd 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/VertexBuilder.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/builder/VertexBuilder.java @@ -56,6 +56,15 @@ public VertexMapping mapping() { @Override public List build(String[] names, Object[] values) { + return this.build(names, values, true); + } + + // Reuse the endpoint identity path, without properties or the ingestion prefilter. + public List buildIdentity(String[] names, Object[] values) { + return this.build(names, values, false); + } + + private List build(String[] names, Object[] values, boolean withProperty) { VertexKVPairs kvPairs = null; // If it's Vertex OLAP properties, VertexOlapKVPairs parsing is needed if (this.verifyOlapVertexBuilder()) { @@ -67,7 +76,7 @@ public List build(String[] names, Object[] values) { kvPairs.headerCaseSensitive(this.headerCaseSensitive()); kvPairs.extractFromVertex(names, values); - return kvPairs.buildVertices(true); + return kvPairs.buildVertices(withProperty); } @Override diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphDeserialization.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphDeserialization.java index 7bed7774d..2a4869c42 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphDeserialization.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphDeserialization.java @@ -32,7 +32,7 @@ import org.apache.hugegraph.util.Log; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.ververica.cdc.debezium.DebeziumDeserializationSchema; +import org.apache.flink.cdc.debezium.DebeziumDeserializationSchema; import io.debezium.data.Envelope; @@ -42,6 +42,10 @@ public class HugeGraphDeserialization implements DebeziumDeserializationSchema collector) { + // Debezium can emit a tombstone after a delete. It has no graph mutation. + if (sourceRecord.value() == null) { + return; + } ObjectMapper mapper = new ObjectMapper(); ObjectNode result = mapper.createObjectNode(); @@ -62,20 +66,37 @@ public void deserialize(SourceRecord sourceRecord, Collector collector) throw new IllegalArgumentException( "The type of `op` should be 'c' 'r' 'u' 'd' only"); } + result.set(Constants.CDC_DATA, row(mapper, data)); + if (operation == Envelope.Operation.UPDATE) { + Struct before = value.getStruct("before"); + if (before == null) { + throw new IllegalArgumentException("CDC UPDATE requires a full before image"); + } + result.set("before", row(mapper, before)); + } + result.put(Constants.CDC_OP, op); + LOG.debug("Loaded CDC event: op={}, dataFields={}, beforeFields={}", op, + result.get(Constants.CDC_DATA).size(), + result.has("before") ? result.get("before").size() : 0); + collector.collect(result.toString()); + } + + private static ObjectNode row(ObjectMapper mapper, Struct data) { ObjectNode rootNode = mapper.createObjectNode(); if (data != null) { Schema afterSchema = data.schema(); List afterFields = afterSchema.fields(); for (Field field : afterFields) { Object afterValue = data.get(field); - rootNode.put(field.name(), afterValue.toString()); + if (afterValue == null) { + rootNode.putNull(field.name()); + } else { + rootNode.put(field.name(), afterValue.toString()); + } } } - result.set(Constants.CDC_DATA, rootNode); - result.put(Constants.CDC_OP, op); - LOG.debug("Loaded data: {}", result); - collector.collect(result.toString()); + return rootNode; } @Override diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphFlinkCDCLoader.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphFlinkCDCLoader.java index 09ccded6d..f766bc591 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphFlinkCDCLoader.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphFlinkCDCLoader.java @@ -33,7 +33,8 @@ import org.slf4j.Logger; import org.apache.hugegraph.util.Log; -import com.ververica.cdc.connectors.mysql.source.MySqlSource; +import org.apache.hugegraph.util.E; +import org.apache.flink.cdc.connectors.mysql.source.MySqlSource; public class HugeGraphFlinkCDCLoader { @@ -45,15 +46,18 @@ public class HugeGraphFlinkCDCLoader { public HugeGraphFlinkCDCLoader(String[] args) { this.options = args; this.loadOptions = LoadOptions.parseOptions(args); + HugeGraphOutputFormat.validateOptions(this.loadOptions); + E.checkArgument(this.loadOptions.sinkParallelism == 1, + "CDC ordered writes require --cdc-sink-parallelism 1"); } public static void main(String[] args) { HugeGraphFlinkCDCLoader loader; try { loader = new HugeGraphFlinkCDCLoader(args); - } catch (Throwable e) { + } catch (RuntimeException e) { Printer.printError("Failed to start loading", e); - return; + throw e; } loader.load(); } @@ -67,10 +71,7 @@ public void load() { MySqlSource mysqlSource = buildMysqlSource(struct); DataStreamSource source = env.fromSource(mysqlSource, WatermarkStrategy.noWatermarks(), "MySQL Source"); - - HugeGraphOutputFormat format = new HugeGraphOutputFormat<>(struct, options); - source.addSink(new HugeGraphSinkFunction<>(format)) - .setParallelism(this.loadOptions.sinkParallelism); + this.configureSink(source, struct); } env.enableCheckpointing(3000); try { @@ -80,6 +81,12 @@ public void load() { } } + void configureSink(DataStreamSource source, InputStruct struct) { + source.setParallelism(1); + HugeGraphOutputFormat format = new HugeGraphOutputFormat<>(struct, options); + source.sinkTo(new HugeGraphSink<>(format)).setParallelism(1); + } + private MySqlSource buildMysqlSource(InputStruct struct) { JDBCSource input = (JDBCSource) struct.input(); String url = input.url(); diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphOutputFormat.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphOutputFormat.java index 2f5e0680e..467a95c8c 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphOutputFormat.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphOutputFormat.java @@ -17,20 +17,25 @@ package org.apache.hugegraph.loader.flink; +import java.io.IOException; +import java.io.ObjectInputStream; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.flink.api.common.io.ParseException; -import org.apache.flink.api.common.io.RichOutputFormat; +import org.apache.flink.api.common.io.OutputFormat; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.util.ExecutorThreadFactory; +import org.apache.flink.util.concurrent.ExecutorThreadFactory; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; @@ -45,9 +50,13 @@ import org.apache.hugegraph.loader.mapping.ElementMapping; import org.apache.hugegraph.loader.mapping.InputStruct; import org.apache.hugegraph.loader.mapping.VertexMapping; +import org.apache.hugegraph.loader.util.JsonUtil; import org.slf4j.Logger; +import org.apache.hugegraph.api.graph.GraphAPI; import org.apache.hugegraph.driver.GraphManager; +import org.apache.hugegraph.structure.constant.Direction; +import org.apache.hugegraph.exception.ServerException; import org.apache.hugegraph.structure.GraphElement; import org.apache.hugegraph.structure.graph.BatchEdgeRequest; import org.apache.hugegraph.structure.graph.BatchVertexRequest; @@ -55,29 +64,48 @@ import org.apache.hugegraph.structure.graph.UpdateStrategy; import org.apache.hugegraph.structure.graph.Vertex; import org.apache.hugegraph.util.Log; +import org.apache.hugegraph.util.E; import io.debezium.data.Envelope; -public class HugeGraphOutputFormat extends RichOutputFormat { +public class HugeGraphOutputFormat implements OutputFormat { private static final Logger LOG = Log.logger(HugeGraphOutputFormat.class); private static final long serialVersionUID = -4514164348993670086L; - private LoadContext loadContext; + private transient LoadContext loadContext; private transient ScheduledExecutorService scheduler; private transient ScheduledFuture scheduledFuture; private transient volatile boolean closed = false; - private final LoadOptions loadOptions; - private final InputStruct struct; - private Map> builders; + // Ship validated configuration, not submitter-side files or live client objects. + private final String optionsJson; + private final String structJson; + private transient LoadOptions loadOptions; + private transient InputStruct struct; + private transient Map> builders; public HugeGraphOutputFormat(InputStruct struct, String[] args) { this.struct = struct; this.loadOptions = LoadOptions.parseOptions(args); + validateOptions(this.loadOptions); + this.optionsJson = JsonUtil.toJson(this.loadOptions); + this.structJson = JsonUtil.toJson(struct); + } + + static void validateOptions(LoadOptions options) { + E.checkArgument(!options.dryRun, "CDC does not support --dry-run true"); + E.checkArgument(!options.usePrefilter, "CDC does not support --use-prefilter true"); + } + + private void readObject(ObjectInputStream input) throws IOException, ClassNotFoundException { + input.defaultReadObject(); + // Parsing CLI options here would require the submitter's mapping file on every worker. + this.loadOptions = JsonUtil.fromJson(this.optionsJson, LoadOptions.class); + this.struct = JsonUtil.fromJson(this.structJson, InputStruct.class); } private Map> initBuilders() { - LoadContext loadContext = new LoadContext(this.loadOptions); + LoadContext loadContext = this.loadContext; Map> builders = new HashMap<>(); for (VertexMapping vertexMapping : this.struct.vertices()) { builders.put(new VertexBuilder(loadContext, this.struct, vertexMapping), @@ -97,9 +125,10 @@ public void configure(Configuration configuration) { } @Override - public void open(int taskNumber, int numTasks) { - this.builders = initBuilders(); + public void open(InitializationContext context) { + validateOptions(this.loadOptions); this.loadContext = new LoadContext(this.loadOptions); + this.builders = initBuilders(); int flushIntervalMs = this.loadOptions.flushIntervalMs; if (flushIntervalMs > 0) { this.scheduler = new ScheduledThreadPoolExecutor(1, new ExecutorThreadFactory( @@ -109,7 +138,7 @@ public void open(int taskNumber, int numTasks) { } } - private synchronized void flushAll() { + synchronized void flushAll() { if (this.closed) { return; } @@ -127,6 +156,9 @@ private synchronized void flushAll() { @Override public synchronized void writeRecord(T row) { + String record = row.toString(); + // Validate every mapping before buffering or writing any part of this event. + validateUpdateIdentities(record); for (Map.Entry> builder : this.builders.entrySet()) { ElementMapping elementMapping = builder.getKey().mapping(); @@ -136,7 +168,7 @@ public synchronized void writeRecord(T row) { // Add batch List graphElements = builder.getValue(); - graphElements.add(row.toString()); + graphElements.add(record); //if (graphElements.size() >= elementMapping.batchSize()) { // flush(builder.getKey(), builder.getValue()); //} @@ -152,14 +184,41 @@ private Tuple2> buildGraphData(ElementBuilder element } catch (JsonProcessingException e) { throw new ParseException(row, e); } - JsonNode data = node.get(Constants.CDC_DATA); String op = node.get(Constants.CDC_OP).asText(); - String[] fields = this.struct.input().header(); - String[] values = new String[data.size()]; - for (int i = 0; i < fields.length; i++) { - values[i] = data.get(fields[i]).asText(); + return Tuple2.of(op, buildElements(elementBuilder, node.get(Constants.CDC_DATA), + Envelope.Operation.DELETE.code().equals(op))); + } + + private List buildElements(ElementBuilder elementBuilder, JsonNode data, + boolean identityOnly) { + if (data == null || !data.isObject()) { + throw new LoadException("CDC event requires a full row image"); } - return Tuple2.of(op, elementBuilder.build(fields, values)); + ElementMapping mapping = elementBuilder.mapping(); + Set nullableKeys = mapping.type().isVertex() ? + this.loadContext.schemaCache().getVertexLabel(mapping.label()).nullableKeys() : + this.loadContext.schemaCache().getEdgeLabel(mapping.label()).nullableKeys(); + List fields = new ArrayList<>(); + List values = new ArrayList<>(); + for (String name : this.struct.input().header()) { + JsonNode value = data.get(name); + if (value == null) { + throw new LoadException("CDC event is missing mapped field '%s'", name); + } + // SQL NULL is structural CDC data, independent of textual null_values sentinels. + String property = mapping.mappingField(name, this.struct.input().headerCaseSensitive()); + if (value.isNull() && nullableKeys.contains(property)) { + continue; + } + fields.add(name); + values.add(value.isNull() ? null : value.asText()); + } + String[] names = fields.toArray(new String[0]); + String[] row = values.toArray(new String[0]); + if (identityOnly && elementBuilder instanceof VertexBuilder) { + return (List) (List) ((VertexBuilder) elementBuilder).buildIdentity(names, row); + } + return elementBuilder.build(names, row); } private void flush(ElementBuilder elementBuilder, List rows) { @@ -179,6 +238,8 @@ private void flush(ElementBuilder elementBuilder, List row } break; case UPDATE: + Set writtenProperties = new HashSet<>(); + graphElements.forEach(element -> writtenProperties.addAll(element.properties().keySet())); Map updateStrategyMap = elementMapping.updateStrategies(); if (isVertex) { @@ -186,22 +247,21 @@ private void flush(ElementBuilder elementBuilder, List row req.vertices((List) (Object) graphElements) .updatingStrategies(updateStrategyMap) .createIfNotExist(true); - g.updateVertices(req.build()); + BatchVertexRequest request = req.build(); + removeNullProperties(g.updateVertices(request), elementMapping, row, writtenProperties); } else { BatchEdgeRequest.Builder req = new BatchEdgeRequest.Builder(); req.edges((List) (Object) graphElements) .updatingStrategies(updateStrategyMap) .checkVertex(this.loadOptions.checkVertex) .createIfNotExist(true); - g.updateEdges(req.build()); + BatchEdgeRequest request = req.build(); + removeNullProperties(g.updateEdges(request), elementMapping, row, writtenProperties); } break; case DELETE: - String id = graphElements.get(0).id().toString(); - if (isVertex) { - g.removeVertex(id); - } else { - g.removeEdge(id); + for (GraphElement element : graphElements) { + removeElement(g, elementBuilder, element); } break; default: @@ -212,6 +272,137 @@ private void flush(ElementBuilder elementBuilder, List row rows.clear(); } + private void validateUpdateIdentities(String row) { + JsonNode node; + try { + node = new ObjectMapper().readTree(row); + } catch (JsonProcessingException e) { + throw new ParseException(row, e); + } + if (!Envelope.Operation.UPDATE.code().equals(node.path(Constants.CDC_OP).asText())) { + return; + } + for (ElementBuilder builder : this.builders.keySet()) { + if (builder.mapping().skip()) { + continue; + } + List before = buildElements(builder, node.get("before"), true); + List unmatched = new ArrayList<>( + buildElements(builder, node.get(Constants.CDC_DATA), true)); + boolean retained = before.size() == unmatched.size(); + for (GraphElement old : before) { + int matching = -1; + for (int i = 0; i < unmatched.size(); i++) { + if (sameIdentity(builder, old, unmatched.get(i))) { + matching = i; + break; + } + } + if (matching < 0) { + retained = false; + break; + } + unmatched.remove(matching); + } + E.checkArgument(retained && unmatched.isEmpty(), + "CDC UPDATE cannot change identity for label '%s'; " + + "vertex IDs, edge endpoints and sort keys must remain unchanged", + builder.mapping().label()); + } + } + + private static boolean sameIdentity(ElementBuilder builder, GraphElement left, GraphElement right) { + if (builder instanceof VertexBuilder) { + return GraphAPI.formatVertexId(left.id()).equals(GraphAPI.formatVertexId(right.id())); + } + Edge old = (Edge) left; + Edge current = (Edge) right; + EdgeBuilder edges = (EdgeBuilder) builder; + return GraphAPI.formatVertexId(old.sourceId()).equals(GraphAPI.formatVertexId(current.sourceId())) && + GraphAPI.formatVertexId(old.targetId()).equals(GraphAPI.formatVertexId(current.targetId())) && + edges.identityProperties(old).equals(edges.identityProperties(current)); + } + + private static void removeElement(GraphManager graph, ElementBuilder builder, GraphElement element) { + try { + if (builder instanceof VertexBuilder) { + graph.getVertex(element.id()); + graph.removeVertex(element.id()); + } else { + Edge edge = (Edge) element; + Map properties = ((EdgeBuilder) builder).identityProperties(edge); + // Resolve the server-owned ID from endpoints, label and sort keys. Keep literal + // strings such as "P.gt(1)" instead of interpreting them as query predicates. + int offset = 0; + int pageSize = 100; + while (true) { + List candidates = graph.getEdges(edge.sourceId(), Direction.OUT, edge.label(), + properties, true, offset, pageSize); + for (Edge candidate : candidates) { + if (sameReturnedVertexId(edge.targetId(), candidate.targetId())) { + graph.removeEdge(candidate.id()); + return; + } + } + if (candidates.size() < pageSize) { + return; + } + offset += pageSize; + } + } + } catch (ServerException e) { + // A delete can be replayed after restoring an earlier checkpoint. + if (e.status() != 404) { + throw e; + } + } + } + + private static boolean sameReturnedVertexId(Object expected, Object actual) { + // Graph API JSON returns UUID endpoint IDs as strings; the builder knows their schema type. + if (expected instanceof UUID && actual instanceof String) { + actual = UUID.fromString((String) actual); + } + return GraphAPI.formatVertexId(expected).equals(GraphAPI.formatVertexId(actual)); + } + + private void removeNullProperties(List elements, + ElementMapping mapping, String row, Set writtenProperties) { + JsonNode data; + try { + data = new ObjectMapper().readTree(row).get(Constants.CDC_DATA); + } catch (JsonProcessingException e) { + throw new ParseException(row, e); + } + boolean caseSensitive = this.struct.input().headerCaseSensitive(); + for (String name : this.struct.input().header()) { + JsonNode value = data.get(name); + if (value == null || (!value.isNull() && !mapping.nullValues().contains(value.asText())) || + containsField(mapping.ignoredFields(), name, caseSensitive) || + (!mapping.selectedFields().isEmpty() && + !containsField(mapping.selectedFields(), name, caseSensitive))) { + continue; + } + String property = mapping.mappingField(name, caseSensitive); + // Another source field may have written the same target, or a non-nullable + // property may retain a configured null sentinel. Trust the actual builder output. + if (writtenProperties.contains(property)) { + continue; + } + for (GraphElement element : elements) { + // Batch update omits null properties; explicitly remove the previous value. + // Returned elements have their GraphManager attached by the client. + if (element.properties().containsKey(property)) { + element.removeProperty(property); + } + } + } + } + + private static boolean containsField(Set fields, String name, boolean caseSensitive) { + return fields.stream().anyMatch(field -> caseSensitive ? field.equals(name) : field.equalsIgnoreCase(name)); + } + @Override public synchronized void close() { if (this.closed) { @@ -222,5 +413,8 @@ public synchronized void close() { this.scheduledFuture.cancel(false); this.scheduler.shutdown(); } + if (this.loadContext != null) { + this.loadContext.client().close(); + } } } diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphSink.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphSink.java new file mode 100644 index 000000000..cad582d91 --- /dev/null +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphSink.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.loader.flink; + +import javax.annotation.Nonnull; + +import org.apache.flink.api.common.io.OutputFormat; +import org.apache.flink.api.connector.sink2.Sink; +import org.apache.flink.api.connector.sink2.SinkWriter; +import org.apache.flink.api.connector.sink2.WriterInitContext; +import org.apache.flink.util.Preconditions; + +public class HugeGraphSink implements Sink { + + private static final long serialVersionUID = -2259171589402599426L; + private final HugeGraphOutputFormat outputFormat; + + public HugeGraphSink(@Nonnull HugeGraphOutputFormat outputFormat) { + this.outputFormat = Preconditions.checkNotNull(outputFormat); + } + + @Override + public SinkWriter createWriter(WriterInitContext context) { + this.outputFormat.open(new OutputFormat.InitializationContext() { + @Override + public int getNumTasks() { + return context.getTaskInfo().getNumberOfParallelSubtasks(); + } + + @Override + public int getTaskNumber() { + return context.getTaskInfo().getIndexOfThisSubtask(); + } + + @Override + public int getAttemptNumber() { + return context.getTaskInfo().getAttemptNumber(); + } + }); + return new SinkWriter() { + @Override + public void write(T value, Context context) { + HugeGraphSink.this.outputFormat.writeRecord(value); + } + + @Override + public void flush(boolean endOfInput) { + HugeGraphSink.this.outputFormat.flushAll(); + } + + @Override + public void close() { + HugeGraphSink.this.outputFormat.close(); + } + }; + } +} diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphSinkFunction.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphSinkFunction.java deleted file mode 100644 index f480feae4..000000000 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/flink/HugeGraphSinkFunction.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hugegraph.loader.flink; - -import javax.annotation.Nonnull; - -import org.apache.flink.api.common.functions.RuntimeContext; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.state.FunctionInitializationContext; -import org.apache.flink.runtime.state.FunctionSnapshotContext; -import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction; -import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; -import org.apache.flink.util.Preconditions; - -public class HugeGraphSinkFunction extends RichSinkFunction - implements CheckpointedFunction { - - private static final long serialVersionUID = -2259171589402599426L; - private final HugeGraphOutputFormat outputFormat; - - public HugeGraphSinkFunction(@Nonnull HugeGraphOutputFormat outputFormat) { - this.outputFormat = Preconditions.checkNotNull(outputFormat); - } - - @Override - public void open(Configuration parameters) throws Exception { - super.open(parameters); - RuntimeContext ctx = getRuntimeContext(); - this.outputFormat.setRuntimeContext(ctx); - this.outputFormat.open(ctx.getIndexOfThisSubtask(), ctx.getNumberOfParallelSubtasks()); - } - - @Override - public void invoke(T value, Context context) { - this.outputFormat.writeRecord(value); - } - - @Override - public void initializeState(FunctionInitializationContext context) { - // pass - } - - @Override - public void snapshotState(FunctionSnapshotContext context) { - // pass - } - - @Override - public void close() throws Exception { - this.outputFormat.close(); - super.close(); - } -} diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/flink/HugeGraphDeserializationTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/flink/HugeGraphDeserializationTest.java new file mode 100644 index 000000000..06ef5c3e1 --- /dev/null +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/flink/HugeGraphDeserializationTest.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.loader.flink; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.flink.util.Collector; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.source.SourceRecord; +import org.apache.log4j.AppenderSkeleton; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.spi.LoggingEvent; +import org.apache.hugegraph.testutil.Assert; +import org.junit.Test; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class HugeGraphDeserializationTest { + + private static final Schema ROW = SchemaBuilder.struct().name("fixture.Row").optional() + .field("id", Schema.INT32_SCHEMA) + .field("name", Schema.STRING_SCHEMA) + .field("city", Schema.OPTIONAL_STRING_SCHEMA).build(); + private static final Schema ENVELOPE = SchemaBuilder.struct().name("fixture.Envelope") + .field("before", ROW).field("after", ROW).field("op", Schema.STRING_SCHEMA).build(); + + @Test + public void testSnapshotAndUpdatePreserveNull() throws Exception { + for (String op : new String[]{"r", "u"}) { + JsonNode event = new ObjectMapper().readTree(convert(record(op, row("old-city"), row(null))).get(0)); + Assert.assertEquals(op, event.get("op").asText()); + Assert.assertEquals("1", event.get("data").get("id").asText()); + Assert.assertTrue(event.get("data").get("city").isNull()); + if ("u".equals(op)) { + Assert.assertEquals("old-city", event.get("before").get("city").asText()); + } + } + } + + @Test + public void testUpdateRequiresBeforeImage() { + Assert.assertThrows(IllegalArgumentException.class, () -> convert(record("u", null, row(null)))); + } + + @Test + public void testDeleteUsesBeforeImage() throws Exception { + JsonNode event = new ObjectMapper().readTree(convert(record("d", row("old-city"), null)).get(0)); + Assert.assertEquals("d", event.get("op").asText()); + Assert.assertEquals("old-city", event.get("data").get("city").asText()); + } + + @Test + public void testTombstoneDoesNotEmitGraphMutation() { + SourceRecord tombstone = new SourceRecord(Collections.emptyMap(), Collections.emptyMap(), + "fixture", null, null); + Assert.assertTrue(convert(tombstone).isEmpty()); + } + + @Test + public void testLogsOnlySafeCdcMetadata() throws Exception { + Logger logger = Logger.getLogger(HugeGraphDeserialization.class.getName()); + Level previous = logger.getLevel(); + CapturingAppender appender = new CapturingAppender(); + logger.setLevel(Level.DEBUG); + logger.addAppender(appender); + try { + JsonNode event = new ObjectMapper().readTree(convert(record( + "u", row("historical-sensitive-value"), row("current-sensitive-value"))).get(0)); + Assert.assertEquals("historical-sensitive-value", event.get("before").get("city").asText()); + Assert.assertEquals("current-sensitive-value", event.get("data").get("city").asText()); + String logged = appender.buffer.toString(); + Assert.assertFalse(logged.contains("historical-sensitive-value")); + Assert.assertFalse(logged.contains("current-sensitive-value")); + Assert.assertFalse(logged.contains("city")); + Assert.assertTrue(logged.contains("op=u")); + Assert.assertTrue(logged.contains("dataFields=3")); + Assert.assertTrue(logged.contains("beforeFields=3")); + } finally { + logger.removeAppender(appender); + logger.setLevel(previous); + } + } + + private static class CapturingAppender extends AppenderSkeleton { + + private final StringBuilder buffer = new StringBuilder(); + + @Override + protected void append(LoggingEvent event) { + this.buffer.append(event.getRenderedMessage()).append('\n'); + } + + @Override + public void close() { + } + + @Override + public boolean requiresLayout() { + return false; + } + } + + private static Struct row(String city) { + return new Struct(ROW).put("id", 1).put("name", "name").put("city", city); + } + + private static SourceRecord record(String op, Struct before, Struct after) { + Struct value = new Struct(ENVELOPE).put("before", before).put("after", after).put("op", op); + return new SourceRecord(Collections.emptyMap(), Collections.emptyMap(), "fixture", ENVELOPE, value); + } + + private static List convert(SourceRecord record) { + List values = new ArrayList<>(); + new HugeGraphDeserialization().deserialize(record, new Collector() { + @Override + public void collect(String value) { + values.add(value); + } + + @Override + public void close() { + } + }); + return values; + } +} diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/flink/HugeGraphIdentityIntegrationTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/flink/HugeGraphIdentityIntegrationTest.java new file mode 100644 index 000000000..fc146b570 --- /dev/null +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/flink/HugeGraphIdentityIntegrationTest.java @@ -0,0 +1,315 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.loader.flink; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.apache.flink.util.Collector; +import org.apache.hugegraph.driver.HugeClient; +import org.apache.hugegraph.driver.SchemaManager; +import org.apache.hugegraph.exception.ServerException; +import org.apache.hugegraph.loader.mapping.InputStruct; +import org.apache.hugegraph.loader.util.JsonUtil; +import org.apache.hugegraph.structure.constant.Direction; +import org.apache.hugegraph.structure.graph.Edge; +import org.apache.hugegraph.structure.graph.Vertex; +import org.apache.hugegraph.testutil.Assert; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.apache.kafka.connect.data.Struct; +import org.apache.kafka.connect.source.SourceRecord; +import org.junit.Assume; +import org.junit.Test; + +/** Explicit opt-in test against a running, test-only HugeGraph instance. */ +public class HugeGraphIdentityIntegrationTest { + + @Test + public void testIdentityChangesDeletesAndReplay() throws Exception { + Assume.assumeTrue(Boolean.getBoolean("flink.identity.integration")); + int port = Integer.getInteger("flink.identity.port", 8080); + String graph = System.getProperty("flink.identity.graph", "hugegraph"); + String prefix = "flink_identity_" + UUID.randomUUID().toString().replace("-", ""); + String custom = prefix + "_custom"; + String primary = prefix + "_primary"; + String edgeLabel = prefix + "_edge"; + String uuidLabel = prefix + "_uuid"; + String name = prefix + "_name"; + String value = prefix + "_value"; + String rank = prefix + "_rank"; + long base = 8100000000000000L + System.nanoTime() % 10000000000L; + List keys = new ArrayList<>(); + List labels = new ArrayList<>(); + List edgeLabels = new ArrayList<>(); + try (HugeClient client = HugeClient.builder("http://127.0.0.1:" + port, + "DEFAULT", graph).build()) { + SchemaManager schema = client.schema(); + try { + for (String key : new String[]{name, value, rank}) { + schema.propertyKey(key).asText().create(); + keys.add(key); + } + schema.vertexLabel(custom).useCustomizeNumberId().properties(value) + .nullableKeys(value).create(); + labels.add(custom); + schema.vertexLabel(primary).usePrimaryKeyId().properties(name, value) + .primaryKeys(name).nullableKeys(value).create(); + labels.add(primary); + schema.edgeLabel(edgeLabel).sourceLabel(custom).targetLabel(custom) + .multiTimes().properties(rank, value).sortKeys(rank).nullableKeys(value).create(); + edgeLabels.add(edgeLabel); + schema.vertexLabel(uuidLabel).useCustomizeUuidId().properties(value).nullableKeys(value).create(); + labels.add(uuidLabel); + verifyUuidVertex(port, client, uuidLabel, value); + + verifyVertices(port, client, custom, value, base + 1000, false); + verifyVertices(port, client, primary, value, 0, true); + + List vertices = new ArrayList<>(); + for (int i = 0; i <= 106; i++) { + long vertexId = base + i; + assertNotFound(() -> client.graph().getVertex(vertexId)); + Vertex vertex = new Vertex(custom); + vertex.id(vertexId); + vertex.property(value, "old"); + vertices.add(vertex); + } + client.graph().addVertices(vertices); + List edges = new ArrayList<>(); + for (int i = 1; i <= 105; i++) { + Edge edge = new Edge(edgeLabel); + edge.source(vertices.get(0)); + edge.target(vertices.get(i)); + edge.property(rank, "P.gt(1)"); + edges.add(edge); + } + client.graph().addEdges(edges); + Map vertexMapping = Map.of("label", custom, "id", "key", + "field_mapping", Map.of("value", value)); + try (FormatFixture fixture = new FormatFixture(port, new String[]{"key", "value"}, + "vertices", vertexMapping)) { + Map before = Map.of("key", Long.toString(base), "value", "old"); + Map after = Map.of("key", Long.toString(base + 10000), "value", "new"); + Assert.assertThrows(IllegalArgumentException.class, () -> fixture.emit("u", before, after)); + Assert.assertEquals("old", client.graph().getVertex(base).property(value)); + Assert.assertEquals(105, client.graph().getEdges(base, Direction.OUT, edgeLabel).size()); + assertNotFound(() -> client.graph().getVertex(base + 10000)); + } + Map mapping = Map.of("label", edgeLabel, + "source", List.of("source"), "target", List.of("target"), + "field_mapping", Map.of("rank", rank, "value", value), + "update_strategies", Map.of(value, "OVERRIDE")); + try (FormatFixture fixture = new FormatFixture(port, + new String[]{"source", "target", "rank", "value"}, "edges", mapping)) { + List ordered = client.graph().getEdges(base, Direction.OUT, edgeLabel, + Map.of(rank, "P.gt(1)"), true, 0, 200); + Assert.assertEquals(105, ordered.size()); + // Choose an actual result beyond the first page instead of assuming ID ordering. + Object lastTarget = ordered.get(104).targetId(); + Map deleted = edgeRow(base, lastTarget, "P.gt(1)", "old"); + fixture.emit("d", deleted, null); + fixture.emit("d", deleted, null); + Assert.assertEquals(104, client.graph().getEdges(base, Direction.OUT, edgeLabel).size()); + assertMissing(client, ordered.get(104)); + + Edge changed = ordered.get(103); + Map before = edgeRow(base, changed.targetId(), "P.gt(1)", "old"); + Map after = edgeRow(base, base + 106, "P.gt(1)", "new"); + Assert.assertThrows(IllegalArgumentException.class, () -> fixture.emit("u", before, after)); + Assert.assertThrows(IllegalArgumentException.class, () -> fixture.emit("u", before, after)); + Assert.assertEquals("P.gt(1)", client.graph().getEdge(changed.id()).property(rank)); + Assert.assertEquals(104, client.graph().getEdges(base, Direction.OUT, edgeLabel).size()); + Assert.assertTrue(client.graph().getEdges(base, Direction.OUT, edgeLabel).stream().noneMatch( + edge -> edge.targetId().toString().equals(Long.toString(base + 106)))); + + Map reranked = edgeRow(base, changed.targetId(), "later", "latest"); + Assert.assertThrows(IllegalArgumentException.class, + () -> fixture.emit("u", before, reranked)); + Assert.assertEquals("P.gt(1)", client.graph().getEdge(changed.id()).property(rank)); + Map updated = edgeRow(base, changed.targetId(), "P.gt(1)", "latest"); + fixture.emit("u", before, updated); + fixture.emit("u", before, updated); + Assert.assertEquals("latest", client.graph().getEdge(changed.id()).property(value)); + fixture.emit("d", updated, null); + fixture.emit("d", updated, null); + assertMissing(client, changed); + Assert.assertEquals(103, client.graph().getEdges(base, Direction.OUT, edgeLabel).size()); + } + } finally { + for (String edge : edgeLabels) { + schema.removeEdgeLabel(edge); + } + for (String label : labels) { + schema.removeVertexLabel(label); + } + for (String key : keys) { + schema.removePropertyKey(key); + } + } + for (String label : labels) { + assertNotFound(() -> schema.getVertexLabel(label)); + } + for (String edge : edgeLabels) { + assertNotFound(() -> schema.getEdgeLabel(edge)); + } + for (String key : keys) { + assertNotFound(() -> schema.getPropertyKey(key)); + } + } + } + + private static void verifyUuidVertex(int port, HugeClient client, String label, + String valueKey) throws Exception { + UUID id = UUID.randomUUID(); + Map mapping = Map.of("label", label, "id", "id", + "field_mapping", Map.of("value", valueKey)); + try (FormatFixture fixture = new FormatFixture(port, new String[]{"id", "value"}, "vertices", mapping)) { + Map row = Map.of("id", id.toString(), "value", "uuid"); + fixture.emit("c", null, row); + Assert.assertEquals("uuid", client.graph().getVertex(id).property(valueKey)); + fixture.emit("d", row, null); + fixture.emit("d", row, null); + assertNotFound(() -> client.graph().getVertex(id)); + } + } + + private static void verifyVertices(int port, HugeClient client, String label, + String valueKey, long id, boolean primary) throws Exception { + String key = primary ? label.replace("_primary", "_name") : "id"; + Map mapping = new java.util.HashMap<>(); + mapping.put("label", label); + mapping.put("update_strategies", Map.of(valueKey, "OVERRIDE")); + mapping.put("field_mapping", Map.of("key", key, "value", valueKey)); + if (!primary) { + mapping.put("id", "key"); + } + if (!primary) { + assertNotFound(() -> client.graph().getVertex(id)); + assertNotFound(() -> client.graph().getVertex(id + 1)); + } + try (FormatFixture fixture = new FormatFixture(port, new String[]{"key", "value"}, + "vertices", mapping)) { + Map before = Map.of("key", primary ? "old" : Long.toString(id), "value", "old"); + Map after = Map.of("key", primary ? "new" : Long.toString(id + 1), "value", "new"); + fixture.emit("c", null, before); + fixture.emit("c", null, before); + List vertices = client.graph().listVertices(label, null, 0, 10); + Assert.assertEquals(1, vertices.size()); + Object oldId = vertices.get(0).id(); + Assert.assertThrows(IllegalArgumentException.class, () -> fixture.emit("u", before, after)); + Assert.assertThrows(IllegalArgumentException.class, () -> fixture.emit("u", before, after)); + Assert.assertEquals("old", client.graph().getVertex(oldId).property(valueKey)); + vertices = client.graph().listVertices(label, null, 0, 10); + Assert.assertEquals(1, vertices.size()); + Map updated = Map.of("key", before.get("key"), "value", "new"); + fixture.emit("u", before, updated); + fixture.emit("u", before, updated); + Assert.assertEquals("new", client.graph().getVertex(oldId).property(valueKey)); + fixture.emit("d", updated, null); + fixture.emit("d", updated, null); + assertNotFound(() -> client.graph().getVertex(oldId)); + Assert.assertTrue(client.graph().listVertices(label, null, 0, 10).isEmpty()); + } + } + + private static Map edgeRow(long source, Object target, String rank, String value) { + return Map.of("source", Long.toString(source), "target", target.toString(), "rank", rank, "value", value); + } + + private static void assertMissing(HugeClient client, Edge edge) { + assertNotFound(() -> client.graph().getEdge(edge.id())); + } + + private static void assertNotFound(Runnable read) { + try { + read.run(); + Assert.fail("Expected HTTP 404"); + } catch (ServerException e) { + Assert.assertEquals(404, e.status()); + } + } + + private static class FormatFixture implements AutoCloseable { + + private final HugeGraphOutputFormat format; + private final Path file; + private final Schema rowSchema; + private final Schema envelope; + private final String[] fields; + + FormatFixture(int port, String[] fields, String kind, Map mapping) throws Exception { + this.fields = fields; + Map input = Map.of("type", "jdbc", "vendor", "mysql", "header", fields); + InputStruct struct = JsonUtil.fromJson(JsonUtil.toJson(Map.of("input", input, + kind, List.of(mapping))), InputStruct.class); + this.file = Files.createTempFile("flink-identity-", ".json"); + this.format = new HugeGraphOutputFormat<>(struct, new String[]{"--file", this.file.toString(), + "--host", "127.0.0.1", "--port", Integer.toString(port), + "--graph", System.getProperty("flink.identity.graph", "hugegraph")}); + this.format.open(null); + SchemaBuilder row = SchemaBuilder.struct().name("fixture.Row").optional(); + for (String field : fields) { + row.field(field, Schema.OPTIONAL_STRING_SCHEMA); + } + this.rowSchema = row.build(); + this.envelope = SchemaBuilder.struct().name("fixture.Envelope") + .field("before", this.rowSchema).field("after", this.rowSchema) + .field("op", Schema.STRING_SCHEMA).build(); + } + + void emit(String op, Map before, Map after) { + Struct value = new Struct(this.envelope).put("op", op) + .put("before", row(before)).put("after", row(after)); + SourceRecord source = new SourceRecord(Collections.emptyMap(), Collections.emptyMap(), + "fixture", this.envelope, value); + new HugeGraphDeserialization().deserialize(source, new Collector() { + @Override + public void collect(String record) { + format.writeRecord(record); + } + + @Override + public void close() { + } + }); + } + + private Struct row(Map values) { + if (values == null) { + return null; + } + Struct row = new Struct(this.rowSchema); + for (String field : this.fields) { + row.put(field, values.get(field)); + } + return row; + } + + @Override + public void close() throws Exception { + this.format.close(); + Files.deleteIfExists(this.file); + } + } +} diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/flink/HugeGraphSinkTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/flink/HugeGraphSinkTest.java new file mode 100644 index 000000000..1f4056bb1 --- /dev/null +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/flink/HugeGraphSinkTest.java @@ -0,0 +1,714 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.loader.flink; + +import java.io.InputStream; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.lang.reflect.Proxy; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import org.apache.flink.api.common.TaskInfo; +import org.apache.flink.api.connector.sink2.Sink; +import org.apache.flink.api.connector.sink2.SinkWriter; +import org.apache.flink.api.connector.sink2.WriterInitContext; +import org.apache.flink.streaming.api.datastream.DataStreamSource; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.functions.source.legacy.ParallelSourceFunction; +import org.apache.flink.streaming.api.graph.StreamGraph; +import org.apache.flink.streaming.api.graph.StreamNode; +import org.apache.flink.streaming.runtime.operators.sink.SinkWriterOperatorFactory; +import org.apache.hugegraph.loader.builder.EdgeBuilder; +import org.apache.hugegraph.loader.builder.SchemaCache; +import org.apache.hugegraph.loader.builder.ElementBuilder; +import org.apache.hugegraph.loader.builder.VertexBuilder; +import org.apache.hugegraph.loader.executor.LoadContext; +import org.apache.hugegraph.loader.executor.LoadOptions; +import org.apache.hugegraph.loader.mapping.EdgeMapping; +import org.apache.hugegraph.loader.mapping.InputStruct; +import org.apache.hugegraph.loader.mapping.LoadMapping; +import org.apache.hugegraph.loader.mapping.ElementMapping; +import org.apache.hugegraph.loader.mapping.VertexMapping; +import org.apache.hugegraph.loader.util.JsonUtil; +import org.apache.hugegraph.driver.GraphManager; +import org.apache.hugegraph.driver.HugeClient; +import org.apache.hugegraph.structure.graph.BatchEdgeRequest; +import org.apache.hugegraph.structure.graph.BatchVertexRequest; +import org.apache.hugegraph.structure.graph.Edge; +import org.apache.hugegraph.structure.constant.Direction; +import org.apache.hugegraph.structure.graph.Vertex; +import org.apache.hugegraph.structure.GraphElement; +import org.apache.hugegraph.structure.constant.IdStrategy; +import org.apache.hugegraph.structure.schema.EdgeLabel; +import org.apache.hugegraph.structure.schema.PropertyKey; +import org.apache.hugegraph.structure.schema.VertexLabel; +import org.apache.hugegraph.testutil.Assert; +import org.junit.Test; + +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; + +public class HugeGraphSinkTest { + + @Test + public void testRejectsUnsupportedOptionsBeforeOpeningClient() throws Exception { + Path mapping = Files.createTempFile("flink-unsupported", ".json"); + try { + Files.writeString(mapping, "not a mapping"); + for (String option : new String[]{"--dry-run", "--use-prefilter"}) { + String[] args = {"--file", mapping.toString(), "--graph", "fixture", option, "true"}; + Assert.assertThrows(IllegalArgumentException.class, () -> new HugeGraphFlinkCDCLoader(args)); + Assert.assertThrows(IllegalArgumentException.class, + () -> new HugeGraphOutputFormat<>(InputStruct.EMPTY, args)); + } + } finally { + Files.deleteIfExists(mapping); + } + } + + @Test + public void testInvalidUpdatePreservesSnapshotIdentity() throws Exception { + try (MappedFixture fixture = new MappedFixture(true)) { + RecordingGraph graph = installRecordingGraph(fixture.format, fixture.builder); + String before = "{\"id\":\"one\",\"a\":\"old\",\"b\":\"old\"}"; + String after = "{\"id\":\"two\",\"a\":\"new\",\"b\":\"new\"}"; + fixture.format.writeRecord("{\"op\":\"r\",\"data\":" + before + "}"); + Assert.assertEquals(Set.of("one"), graph.vertices.keySet()); + Assert.assertThrows(IllegalArgumentException.class, + () -> fixture.format.writeRecord("{\"op\":\"u\",\"before\":" + before + + ",\"data\":" + after + "}")); + Assert.assertEquals(0, graph.deletes); + Assert.assertEquals(Set.of("one"), graph.vertices.keySet()); + Assert.assertEquals("old", graph.vertices.get("one").property("city")); + } + } + + @Test + public void testInvalidEdgeUpdatePreservesSnapshotIdentity() throws Exception { + try (MappedFixture fixture = new MappedFixture(true)) { + InputStruct struct = JsonUtil.fromJson("{\"input\":{\"type\":\"jdbc\",\"vendor\":\"mysql\", " + + "\"header\":[\"source\",\"target\",\"city\"]}}", InputStruct.class); + EdgeLabel label = JsonUtil.fromJson("{\"name\":\"fixture_edge\",\"frequency\":\"SINGLE\", " + + "\"links\":[{\"fixture\":\"fixture\"}]," + + "\"properties\":[\"city\"],\"nullable_keys\":[\"city\"]}", + EdgeLabel.class); + Field contextField = HugeGraphOutputFormat.class.getDeclaredField("loadContext"); + contextField.setAccessible(true); + LoadContext context = (LoadContext) contextField.get(fixture.format); + Field cacheField = LoadContext.class.getDeclaredField("schemaCache"); + cacheField.setAccessible(true); + cacheField.set(context, new SchemaCache(List.of(new PropertyKey("city")), + List.of(fixture.label), List.of(label))); + EdgeMapping mapping = new EdgeMapping(List.of("source"), false, List.of("target"), false); + mapping.label("fixture_edge"); + EdgeBuilder builder = new EdgeBuilder(context, struct, mapping); + HugeGraphOutputFormat format = new HugeGraphOutputFormat<>(struct, + new String[]{"--file", fixture.file.toString(), "--graph", "fixture"}); + contextField.set(format, context); + RecordingGraph graph = installRecordingGraph(format, builder); + String before = "{\"source\":\"one\",\"target\":\"two\",\"city\":\"old\"}"; + String after = "{\"source\":\"one\",\"target\":\"three\",\"city\":\"new\"}"; + format.writeRecord("{\"op\":\"r\",\"data\":" + before + "}"); + Assert.assertEquals(1, graph.edges.size()); + Assert.assertThrows(IllegalArgumentException.class, + () -> format.writeRecord("{\"op\":\"u\",\"before\":" + before + + ",\"data\":" + after + "}")); + Assert.assertEquals(0, graph.edgeReads); + Assert.assertEquals(0, graph.deletes); + Assert.assertEquals(1, graph.edges.size()); + Assert.assertEquals("two", graph.edges.get(0).targetId()); + Assert.assertEquals("old", graph.edges.get(0).property("city")); + } + } + + @Test + public void testAllMappingsAreCheckedBeforeAnyWriteOrBufferChange() throws Exception { + try (MappedFixture fixture = new MappedFixture(true)) { + RecordingGraph graph = installRecordingGraph(fixture.format, fixture.builder); + Field contextField = HugeGraphOutputFormat.class.getDeclaredField("loadContext"); + contextField.setAccessible(true); + LoadContext context = (LoadContext) contextField.get(fixture.format); + VertexMapping changedMapping = new VertexMapping("a", false); + changedMapping.label("fixture"); + VertexBuilder changed = new VertexBuilder(context, fixture.struct, changedMapping); + Map> mappings = new LinkedHashMap<>(); + mappings.put(fixture.builder, new ArrayList<>()); + mappings.put(changed, new ArrayList<>()); + Field builders = HugeGraphOutputFormat.class.getDeclaredField("builders"); + builders.setAccessible(true); + builders.set(fixture.format, mappings); + String before = "{\"id\":\"one\",\"a\":\"old\",\"b\":\"old\"}"; + String after = "{\"id\":\"one\",\"a\":\"new\",\"b\":\"new\"}"; + Assert.assertThrows(IllegalArgumentException.class, + () -> fixture.format.writeRecord("{\"op\":\"u\",\"before\":" + before + + ",\"data\":" + after + "}")); + Assert.assertTrue(graph.vertices.isEmpty()); + Assert.assertEquals(0, graph.deletes); + mappings.values().forEach(rows -> Assert.assertTrue(rows.isEmpty())); + } + } + + @Test + public void testRejectsAddedAndRemovedUnfoldedIdentities() throws Exception { + for (String[] ids : new String[][]{{"one", "one|two"}, {"one|two", "one"}, + {"one|two", "one|three"}}) { + try (MappedFixture fixture = new MappedFixture(true, true)) { + RecordingGraph graph = installRecordingGraph(fixture.format, fixture.builder); + String before = JsonUtil.toJson(Map.of("id", ids[0], "a", "old", "b", "old")); + String after = JsonUtil.toJson(Map.of("id", ids[1], "a", "new", "b", "new")); + fixture.format.writeRecord("{\"op\":\"r\",\"data\":" + before + "}"); + Set existing = Set.copyOf(graph.vertices.keySet()); + Assert.assertThrows(IllegalArgumentException.class, + () -> fixture.format.writeRecord("{\"op\":\"u\",\"before\":" + before + + ",\"data\":" + after + "}")); + Assert.assertEquals(existing, graph.vertices.keySet()); + graph.vertices.values().forEach(vertex -> Assert.assertEquals("old", vertex.property("city"))); + Assert.assertEquals(0, graph.deletes); + Field builders = HugeGraphOutputFormat.class.getDeclaredField("builders"); + builders.setAccessible(true); + Map> mappings = + (Map>) builders.get(fixture.format); + mappings.values().forEach(rows -> Assert.assertTrue(rows.isEmpty())); + } + } + } + + private static RecordingGraph installRecordingGraph(HugeGraphOutputFormat format, + ElementBuilder builder) throws Exception { + Class unsafe = Class.forName("sun.misc.Unsafe"); + Field instance = unsafe.getDeclaredField("theUnsafe"); + instance.setAccessible(true); + Object allocator = instance.get(null); + // Reuse the cached-schema allocation seam without constructing an HTTP client. + RecordingGraph graph = (RecordingGraph) unsafe.getMethod("allocateInstance", Class.class) + .invoke(allocator, RecordingGraph.class); + graph.vertices = new LinkedHashMap<>(); + graph.edges = new ArrayList<>(); + HugeClient client = (HugeClient) unsafe.getMethod("allocateInstance", Class.class) + .invoke(allocator, HugeClient.class); + Field graphField = HugeClient.class.getDeclaredField("graph"); + graphField.setAccessible(true); + graphField.set(client, graph); + Field contextField = HugeGraphOutputFormat.class.getDeclaredField("loadContext"); + contextField.setAccessible(true); + LoadContext context = (LoadContext) contextField.get(format); + Field clientField = LoadContext.class.getDeclaredField("client"); + clientField.setAccessible(true); + clientField.set(context, client); + Field builders = HugeGraphOutputFormat.class.getDeclaredField("builders"); + builders.setAccessible(true); + builders.set(format, Map.of(builder, new ArrayList())); + return graph; + } + + private static class RecordingGraph extends GraphManager { + + private Map vertices; + private List edges; + private int edgeReads; + private int deletes; + + RecordingGraph() { + super(null, "DEFAULT", "fixture"); + } + + @Override + public List addVertices(List batch) { + batch.forEach(vertex -> this.vertices.put(vertex.id(), vertex)); + return batch; + } + + @Override + public Vertex getVertex(Object id) { + return this.vertices.get(id); + } + + @Override + public void removeVertex(Object id) { + this.deletes++; + this.vertices.remove(id); + } + + @Override + public List updateVertices(BatchVertexRequest request) { + throw new AssertionError("Invalid request reached graph update"); + } + + @Override + public List addEdges(List batch) { + this.edges.addAll(batch); + return batch; + } + + @Override + public List getEdges(Object vertexId, Direction direction, String label, + Map properties, boolean keepP, int offset, int limit) { + this.edgeReads++; + return this.edges; + } + + @Override + public void removeEdge(String id) { + this.deletes++; + this.edges.clear(); + } + + @Override + public List updateEdges(BatchEdgeRequest request) { + throw new AssertionError("Invalid request reached graph update"); + } + } + + @Test + public void testRejectsParallelSinkBeforeReadingMapping() throws Exception { + Path mapping = Files.createTempFile("flink-parallelism", ".json"); + try { + Files.writeString(mapping, "not a mapping"); + for (String parallelism : new String[]{"0", "-1", "2"}) { + Assert.assertThrows(IllegalArgumentException.class, () -> new HugeGraphFlinkCDCLoader( + new String[]{"--file", mapping.toString(), "--graph", "fixture", + "--cdc-sink-parallelism", parallelism})); + } + } finally { + Files.deleteIfExists(mapping); + } + } + + @Test + public void testInvalidParallelSinkFailsDirectJvm() throws Exception { + Path directory = Files.createTempDirectory("flink-exit"); + Process process = null; + try { + Path mapping = directory.resolve("mapping.json"); + Files.writeString(mapping, "not a mapping"); + String classpath = System.getProperty("surefire.test.class.path"); + Assert.assertNotNull(classpath); + Path java = Path.of(System.getProperty("java.home"), "bin", "java"); + process = new ProcessBuilder(java.toString(), "-Xmx96m", "-XX:ActiveProcessorCount=2", + "-Dfile.encoding=UTF-8", "-cp", classpath, + HugeGraphFlinkCDCLoader.class.getName(), + "--file", mapping.toString(), "--graph", "fixture", + "--cdc-sink-parallelism", "2") + .directory(directory.toFile()).redirectErrorStream(true).start(); + Assert.assertTrue("Direct JVM did not exit", process.waitFor(30, TimeUnit.SECONDS)); + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + Assert.assertTrue(output, output.contains("CDC ordered writes require --cdc-sink-parallelism 1")); + Assert.assertFalse(output, output.contains("NoClassDefFoundError")); + Assert.assertFalse(output, output.contains("ClassNotFoundException")); + Assert.assertFalse(output, output.contains("OutOfMemoryError")); + Assert.assertNotEquals(output, 0, process.exitValue()); + } finally { + if (process != null && process.isAlive()) { + process.destroyForcibly(); + } + try (Stream files = Files.walk(directory)) { + for (Path file : files.sorted(Comparator.reverseOrder()).toList()) { + Files.delete(file); + } + } + } + } + + @Test + public void testAcceptsDefaultAndExplicitSerialSink() throws Exception { + Path mapping = Files.createTempFile("flink-parallelism", ".json"); + try { + Assert.assertNotNull(new HugeGraphFlinkCDCLoader( + new String[]{"--file", mapping.toString(), "--graph", "fixture"})); + Assert.assertNotNull(new HugeGraphFlinkCDCLoader( + new String[]{"--file", mapping.toString(), "--graph", "fixture", + "--cdc-sink-parallelism", "1"})); + } finally { + Files.deleteIfExists(mapping); + } + } + + @Test + public void testPipelinePinsSourceAndSinkWithoutChangingGlobalParallelism() throws Exception { + Path mapping = Files.createTempFile("flink-parallelism", ".json"); + try { + try (InputStream resource = getClass().getResourceAsStream("/jdbc_number_to_string/struct.json")) { + Assert.assertNotNull(resource); + Files.copy(resource, mapping, StandardCopyOption.REPLACE_EXISTING); + } + for (boolean globalParallelism : new boolean[]{false, true}) { + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + if (globalParallelism) { + env.setParallelism(2); + } + int configuredParallelism = env.getParallelism(); + HugeGraphFlinkCDCLoader loader = new HugeGraphFlinkCDCLoader( + new String[]{"--file", mapping.toString(), "--graph", "fixture"}); + for (InputStruct struct : LoadMapping.of(mapping.toString()).structs()) { + DataStreamSource source = env.addSource(new ParallelFixtureSource()); + if (globalParallelism) { + Assert.assertEquals(2, source.getParallelism()); + } + loader.configureSink(source, struct); + Assert.assertEquals(1, source.getParallelism()); + } + Assert.assertEquals(configuredParallelism, env.getParallelism()); + StreamGraph graph = env.getStreamGraph(); + Assert.assertEquals(2, graph.getSourceIDs().size()); + Assert.assertEquals(4, graph.getStreamNodes().size()); + int sinks = 0; + for (StreamNode node : graph.getStreamNodes()) { + Assert.assertEquals(1, node.getParallelism()); + if (node.getOperatorFactory() instanceof SinkWriterOperatorFactory) { + SinkWriterOperatorFactory writer = + (SinkWriterOperatorFactory) node.getOperatorFactory(); + Assert.assertTrue(writer.getSink() instanceof HugeGraphSink); + Assert.assertTrue(node.getOutEdges().isEmpty()); + sinks++; + } + } + Assert.assertEquals(2, sinks); + } + } finally { + Files.deleteIfExists(mapping); + } + } + + private static class ParallelFixtureSource implements ParallelSourceFunction { + + private static final long serialVersionUID = 1L; + private volatile boolean cancelled; + + @Override + public void run(SourceContext context) { + synchronized (context.getCheckpointLock()) { + if (!this.cancelled) { + context.collect("fixture"); + } + } + } + + @Override + public void cancel() { + this.cancelled = true; + } + } + + @Test + public void testUsesPublicFlinkWriterContract() throws Exception { + // CDC's compatibility jar also contains a Sink class with a legacy abstract method. + Assert.assertTrue(Modifier.isAbstract(Sink.class.getMethod("createWriter", + WriterInitContext.class).getModifiers())); + } + + @Test + public void testWriterLifecycleAndFlushFailure() throws Exception { + Path mapping = Files.createTempFile("flink-sink", ".json"); + try { + RecordingFormat format = new RecordingFormat(mapping); + HugeGraphSink sink = new HugeGraphSink<>(format); + SinkWriter writer = sink.createWriter(context()); + Assert.assertEquals(2, format.tasks); + Assert.assertEquals(1, format.task); + Assert.assertEquals(3, format.attempt); + writer.write("record", null); + Assert.assertEquals(1, format.rows.size()); + Assert.assertEquals("record", format.rows.get(0)); + writer.flush(false); + Assert.assertEquals(1, format.flushes); + format.failFlush = true; + Assert.assertThrows(IllegalStateException.class, () -> writer.flush(false)); + writer.close(); + Assert.assertTrue(format.closed); + } finally { + Files.deleteIfExists(mapping); + } + } + + @Test + public void testNullUpdateRemovesMappedPropertyAndCanBeReplayed() throws Exception { + Path file = Files.createTempFile("flink-null", ".json"); + try { + InputStruct struct = JsonUtil.fromJson("{\"input\":{\"type\":\"jdbc\",\"vendor\":\"mysql\", " + + "\"header\":[\"city\",\"name\"]}}", + InputStruct.class); + HugeGraphOutputFormat format = new HugeGraphOutputFormat<>(struct, + new String[]{"--file", file.toString(), "--graph", "fixture"}); + VertexMapping mapping = new VertexMapping("id", false); + mapping.mappingFields(Map.of("city", "location")); + mapping.ignoredFields(Set.of("name")); + Vertex vertex = new Vertex("fixture") { + @Override + public Vertex removeProperty(String key) { + Assert.assertTrue(this.properties().containsKey(key)); + this.properties().remove(key); + return this; + } + }; + vertex.property("location", "old"); + vertex.property("name", "keep"); + Method update = HugeGraphOutputFormat.class.getDeclaredMethod("removeNullProperties", + List.class, ElementMapping.class, + String.class, Set.class); + update.setAccessible(true); + String event = "{\"data\":{\"city\":null,\"name\":null}}"; + update.invoke(format, List.of(vertex), mapping, event, Set.of()); + update.invoke(format, List.of(vertex), mapping, event, Set.of()); + Assert.assertNull(vertex.property("location")); + Assert.assertEquals("keep", vertex.property("name")); + } finally { + Files.deleteIfExists(file); + } + } + + @Test + public void testReturnedEndpointIdKeepsSchemaType() throws Exception { + Method match = HugeGraphOutputFormat.class.getDeclaredMethod("sameReturnedVertexId", + Object.class, Object.class); + match.setAccessible(true); + UUID uuid = UUID.randomUUID(); + Assert.assertTrue((boolean) match.invoke(null, uuid, uuid.toString())); + Assert.assertTrue((boolean) match.invoke(null, 123L, 123)); + Assert.assertFalse((boolean) match.invoke(null, 123L, "123")); + } + + @Test + public void testNullAliasKeepsConstructedMappedProperty() throws Exception { + for (String missing : new String[]{null, "", ""}) { + try (MappedFixture fixture = new MappedFixture(true)) { + if ("".equals(missing)) { + fixture.mapping.nullValues(Set.of("")); + } + Assert.assertEquals("Paris", fixture.update(missing, "Paris").property("city")); + Assert.assertEquals("Berlin", fixture.update("Berlin", missing).property("city")); + } + } + } + + @Test + public void testConfiguredNullSentinelsRemoveStaleProperty() throws Exception { + for (String missing : new String[]{null, "", ""}) { + try (MappedFixture fixture = new MappedFixture(true)) { + if ("".equals(missing)) { + fixture.mapping.nullValues(Set.of("")); + } + Assert.assertNull(fixture.update(missing, missing).property("city")); + } + } + } + + @Test + public void testCleanupKeepsNonNullableIgnoredAndUnmappedProperties() throws Exception { + try (MappedFixture fixture = new MappedFixture(false)) { + Assert.assertEquals("", fixture.update("", "").property("city")); + } + for (boolean selected : new boolean[]{false, true}) { + try (MappedFixture fixture = new MappedFixture(true)) { + fixture.mapping.mappingFields(Map.of("a", "city", "b", "other")); + if (selected) { + fixture.mapping.selectedFields(Set.of("id", "b")); + } else { + fixture.mapping.ignoredFields(Set.of("a")); + } + fixture.label.properties().add("other"); + fixture.label.nullableKeys().add("other"); + fixture.unmappedNull = true; + Assert.assertEquals("old", fixture.update(null, "keep").property("city")); + } + } + } + + @Test + public void testSqlNullDoesNotDependOnConfiguredNullSentinels() throws Exception { + try (MappedFixture fixture = new MappedFixture(true)) { + fixture.mapping.nullValues(Set.of()); + Assert.assertNull(fixture.update(null, null).property("city")); + Assert.assertEquals("", fixture.update("", "").property("city")); + Assert.assertEquals("Paris", fixture.update(null, "Paris").property("city")); + Assert.assertEquals("Berlin", fixture.update("Berlin", null).property("city")); + } + try (MappedFixture fixture = new MappedFixture(false)) { + fixture.mapping.nullValues(Set.of()); + Assert.assertThrows(InvocationTargetException.class, () -> fixture.update(null, null), + failure -> Assert.assertTrue(failure.getCause() instanceof IllegalArgumentException)); + } + } + + private static class MappedFixture implements AutoCloseable { + + private final Path file; + private final InputStruct struct; + private final VertexMapping mapping; + private final VertexLabel label; + private final VertexBuilder builder; + private final HugeGraphOutputFormat format; + private boolean unmappedNull; + + MappedFixture(boolean nullable) throws Exception { + this(nullable, false); + } + + MappedFixture(boolean nullable, boolean unfold) throws Exception { + this.file = Files.createTempFile("flink-mapped-null", ".json"); + this.struct = JsonUtil.fromJson("{\"input\":{\"type\":\"jdbc\",\"vendor\":\"mysql\", " + + "\"header\":[\"id\",\"a\",\"b\"],\"list_format\":{\"start_symbol\":\"\"," + + "\"end_symbol\":\"\",\"elem_delimiter\":\"|\"}}}", InputStruct.class); + this.mapping = new VertexMapping("id", unfold); + this.mapping.label("fixture"); + this.mapping.mappingFields(Map.of("a", "city", "b", "city")); + this.label = new VertexLabel("fixture"); + this.label.idStrategy(IdStrategy.CUSTOMIZE_STRING); + this.label.properties().add("city"); + if (nullable) { + this.label.nullableKeys().add("city"); + } + SchemaCache cache = new SchemaCache(List.of(new PropertyKey("city"), new PropertyKey("other")), + List.of(this.label), List.of()); + // Match the existing LoadContextTest allocation seam: cached schema, no clients. + Class unsafe = Class.forName("sun.misc.Unsafe"); + Field instance = unsafe.getDeclaredField("theUnsafe"); + instance.setAccessible(true); + LoadContext context = (LoadContext) unsafe.getMethod("allocateInstance", Class.class) + .invoke(instance.get(null), LoadContext.class); + for (Map.Entry field : Map.of("options", new LoadOptions(), + "schemaCache", cache).entrySet()) { + Field target = LoadContext.class.getDeclaredField(field.getKey()); + target.setAccessible(true); + target.set(context, field.getValue()); + } + this.builder = new VertexBuilder(context, this.struct, this.mapping); + this.format = new HugeGraphOutputFormat<>(this.struct, + new String[]{"--file", this.file.toString(), "--graph", "fixture"}); + Field loadContext = HugeGraphOutputFormat.class.getDeclaredField("loadContext"); + loadContext.setAccessible(true); + loadContext.set(this.format, context); + } + + Vertex update(String first, String second) throws Exception { + Map data = new LinkedHashMap<>(); + data.put("id", "one"); + data.put("a", first); + data.put("b", second); + if (this.unmappedNull) { + // Not in the input header: do not remove an unrelated graph property. + data.put("city", null); + } + Method build = HugeGraphOutputFormat.class.getDeclaredMethod("buildElements", + ElementBuilder.class, JsonNode.class, + boolean.class); + build.setAccessible(true); + List built = (List) build.invoke(this.format, this.builder, + new ObjectMapper().valueToTree(data), false); + Assert.assertEquals(1, built.size()); + Vertex returned = new Vertex("fixture") { + @Override + public Vertex removeProperty(String key) { + this.properties().remove(key); + return this; + } + }; + returned.property("city", "old"); + built.get(0).properties().forEach(returned::property); + Method remove = HugeGraphOutputFormat.class.getDeclaredMethod("removeNullProperties", + List.class, ElementMapping.class, + String.class, Set.class); + remove.setAccessible(true); + remove.invoke(this.format, List.of(returned), this.mapping, + JsonUtil.toJson(Map.of("data", data)), built.get(0).properties().keySet()); + return returned; + } + + @Override + public void close() throws Exception { + Files.deleteIfExists(this.file); + } + } + + private static WriterInitContext context() { + TaskInfo task = (TaskInfo) Proxy.newProxyInstance(TaskInfo.class.getClassLoader(), + new Class[]{TaskInfo.class}, (proxy, method, args) -> { + switch (method.getName()) { + case "getNumberOfParallelSubtasks": + return 2; + case "getIndexOfThisSubtask": + return 1; + case "getAttemptNumber": + return 3; + default: + throw new UnsupportedOperationException(method.getName()); + } + }); + return (WriterInitContext) Proxy.newProxyInstance(WriterInitContext.class.getClassLoader(), + new Class[]{WriterInitContext.class}, (proxy, method, args) -> { + if (method.getName().equals("getTaskInfo")) { + return task; + } + throw new UnsupportedOperationException(method.getName()); + }); + } + + private static class RecordingFormat extends HugeGraphOutputFormat { + + private static final long serialVersionUID = 1L; + private final List rows = new ArrayList<>(); + private int tasks; + private int task; + private int attempt; + private int flushes; + private boolean failFlush; + private boolean closed; + + RecordingFormat(Path mapping) { + super(InputStruct.EMPTY, new String[]{"--file", mapping.toString(), "--graph", "fixture"}); + } + + @Override + public void open(InitializationContext context) { + this.tasks = context.getNumTasks(); + this.task = context.getTaskNumber(); + this.attempt = context.getAttemptNumber(); + } + + @Override + public void writeRecord(Object row) { + this.rows.add(row); + } + + @Override + synchronized void flushAll() { + if (this.failFlush) { + throw new IllegalStateException("flush failed"); + } + this.flushes++; + } + + @Override + public void close() { + this.closed = true; + } + } +} diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/flink/PackagedCdcClassLoadingProbe.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/flink/PackagedCdcClassLoadingProbe.java new file mode 100644 index 000000000..b32cdcd15 --- /dev/null +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/flink/PackagedCdcClassLoadingProbe.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.loader.flink; + +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.ThreadFactory; +import java.util.logging.Logger; +import java.util.stream.Stream; + +/** + * Run against a built job JAR and Flink's lib directory, with no Maven classpath or database. + */ +public final class PackagedCdcClassLoadingProbe { + + private static final Logger LOG = Logger.getLogger(PackagedCdcClassLoadingProbe.class.getName()); + private static final String GUAVA = "org.apache.flink.shaded.guava"; + + private PackagedCdcClassLoadingProbe() { + } + + public static void main(String[] args) throws Exception { + if (args.length != 2) { + throw new IllegalArgumentException("Expected job JAR and Flink lib directory"); + } + Path job = Path.of(args[0]).toRealPath(); + Path libraries = Path.of(args[1]).toRealPath(); + URL[] runtimeJars; + try (Stream files = Files.list(libraries)) { + runtimeJars = files.filter(path -> path.toString().endsWith(".jar")).sorted() + .map(PackagedCdcClassLoadingProbe::url).toArray(URL[]::new); + } + try (URLClassLoader runtime = new URLClassLoader(runtimeJars, ClassLoader.getPlatformClassLoader()); + URLClassLoader userCode = new URLClassLoader(new URL[]{url(job)}, runtime)) { + Class legacy = threadFactory(userCode, "31", "snapshot-splitting"); + if (!origin(legacy).equals(job)) { + throw new AssertionError("CDC Guava31 must come from the packaged job"); + } + Class current = threadFactory(runtime, "33", "flink-runtime"); + if (origin(current).equals(job) || current.getClassLoader() != runtime) { + throw new AssertionError("Flink Guava33 must remain owned by the runtime"); + } + String[] coordinatorClasses = { + "org.apache.flink.cdc.connectors.mysql.source.assigners.MySqlSnapshotSplitAssigner", + "org.apache.flink.cdc.connectors.mysql.source.enumerator.MySqlSourceEnumerator" + }; + for (String name : coordinatorClasses) { + Class coordinator = Class.forName(name, false, userCode); + coordinator.getDeclaredConstructors(); + coordinator.getDeclaredMethods(); + if (!origin(coordinator).equals(job)) { + throw new AssertionError("CDC coordinator must come from the packaged job"); + } + LOG.info(name + " links from " + origin(coordinator)); + } + Class source = Class.forName("org.apache.flink.api.connector.source.Source", false, userCode); + if (source.getClassLoader() != runtime) { + throw new AssertionError("The job must use the runtime's public Flink source API"); + } + LOG.info("Guava31=" + origin(legacy) + ", Guava33=" + origin(current) + + ", FlinkSource=" + origin(source)); + } + } + + private static Class threadFactory(ClassLoader loader, String version, String name) throws Exception { + Class type = Class.forName(GUAVA + version + ".com.google.common.util.concurrent.ThreadFactoryBuilder", + true, loader); + Object builder = type.getConstructor().newInstance(); + type.getMethod("setNameFormat", String.class).invoke(builder, name); + ThreadFactory factory = (ThreadFactory) type.getMethod("build").invoke(builder); + Thread thread = factory.newThread(() -> { }); + if (!thread.getName().equals(name)) { + throw new AssertionError("CDC's thread factory contract failed"); + } + // Creating the factory exercises CDC's failing dependency; never start the thread. + return type; + } + + private static Path origin(Class type) throws Exception { + return Path.of(type.getProtectionDomain().getCodeSource().getLocation().toURI()).toRealPath(); + } + + private static URL url(Path path) { + try { + return path.toUri().toURL(); + } catch (Exception e) { + throw new IllegalArgumentException("Invalid classpath entry", e); + } + } +} diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/FlinkSinkSerializationTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/FlinkSinkSerializationTest.java new file mode 100644 index 000000000..4e1a22c1e --- /dev/null +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/FlinkSinkSerializationTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.loader.test.unit; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +import org.apache.hugegraph.loader.executor.LoadOptions; +import org.apache.hugegraph.loader.flink.HugeGraphOutputFormat; +import org.apache.hugegraph.loader.flink.HugeGraphSink; +import org.apache.hugegraph.loader.mapping.InputStruct; +import org.apache.hugegraph.loader.mapping.LoadMapping; +import org.apache.hugegraph.loader.source.jdbc.JDBCSource; +import org.apache.hugegraph.loader.util.JsonUtil; +import org.apache.hugegraph.testutil.Assert; +import org.junit.Test; + +public class FlinkSinkSerializationTest { + + @Test + public void testJdbcMappingAndOptionsRoundTripWithoutSubmitterFile() throws Exception { + Path mapping = Files.createTempFile("flink-mapping", ".json"); + try { + try (InputStream resource = getClass().getResourceAsStream("/jdbc_number_to_string/struct.json")) { + Assert.assertNotNull(resource); + Files.copy(resource, mapping, StandardCopyOption.REPLACE_EXISTING); + } + InputStruct struct = LoadMapping.of(mapping.toString()).structs().get(0); + HugeGraphOutputFormat format = new HugeGraphOutputFormat<>(struct, new String[]{ + "--file", mapping.toString(), "--graph", "fixture", + "--cdc-flush-interval", "1234", "--cdc-sink-parallelism", "2" + }); + HugeGraphSink sink = new HugeGraphSink<>(format); + Files.delete(mapping); + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + try (ObjectOutputStream output = new ObjectOutputStream(buffer)) { + output.writeObject(sink); + } + Object restored; + try (ObjectInputStream input = new ObjectInputStream(new ByteArrayInputStream(buffer.toByteArray()))) { + restored = input.readObject(); + } + Object restoredFormat = field(restored, "outputFormat"); + LoadOptions options = (LoadOptions) field(restoredFormat, "loadOptions"); + InputStruct restoredStruct = (InputStruct) field(restoredFormat, "struct"); + Assert.assertEquals("fixture", options.graph); + Assert.assertEquals(1234, options.flushIntervalMs); + Assert.assertEquals(2, options.sinkParallelism); + Assert.assertTrue(restoredStruct.input() instanceof JDBCSource); + Assert.assertEquals(JsonUtil.toJson(struct), JsonUtil.toJson(restoredStruct)); + Assert.assertNull(field(restoredFormat, "loadContext")); + Assert.assertNull(field(restoredFormat, "builders")); + } finally { + Files.deleteIfExists(mapping); + } + } + + private static Object field(Object object, String name) throws Exception { + Field field = object.getClass().getDeclaredField(name); + field.setAccessible(true); + return field.get(object); + } +} diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/UnitTestSuite.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/UnitTestSuite.java index aafc4b2e7..432ed671f 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/UnitTestSuite.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/UnitTestSuite.java @@ -30,7 +30,10 @@ DateUtilTest.class, MappingConverterTest.class, LoadProgressTest.class, - RangesTimerTest.class + RangesTimerTest.class, + FlinkSinkSerializationTest.class, + org.apache.hugegraph.loader.flink.HugeGraphSinkTest.class, + org.apache.hugegraph.loader.flink.HugeGraphDeserializationTest.class }) public class UnitTestSuite { } diff --git a/hugegraph-loader/src/test/scripts/flink-launcher.test.cjs b/hugegraph-loader/src/test/scripts/flink-launcher.test.cjs new file mode 100644 index 000000000..c4ab07a58 --- /dev/null +++ b/hugegraph-loader/src/test/scripts/flink-launcher.test.cjs @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { spawnSync } = require('node:child_process'); +const { mkdtempSync, mkdirSync, copyFileSync, writeFileSync, readFileSync, realpathSync, rmSync } = require('node:fs'); +const { tmpdir } = require('node:os'); +const { join, resolve } = require('node:path'); + +test('CDC launcher keeps argument boundaries and does not expose arguments', t => { + const root = mkdtempSync(join(tmpdir(), 'flink-launcher-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + mkdirSync(join(root, 'loader/bin'), { recursive: true }); + mkdirSync(join(root, 'loader/lib')); + mkdirSync(join(root, 'flink/bin'), { recursive: true }); + const script = join(root, 'loader/bin/hugegraph-flinkcdc-loader.sh'); + copyFileSync(resolve(__dirname, '../../../assembly/static/bin/hugegraph-flinkcdc-loader.sh'), script); + const argsFile = join(root, 'args'); + writeFileSync(join(root, 'flink/bin/flink'), '#!/bin/bash\nprintf "%s\\0" "$@" > "$PROBE_ARGS"\n', { mode: 0o755 }); + const jar = join(root, 'loader/lib/apache-hugegraph-loader-1.8.0-shaded.jar'); + const run = (args, extra = {}) => spawnSync('bash', [script, ...args], { + env: { ...process.env, FLINK_HOME: join(root, 'flink'), PROBE_ARGS: argsFile, ...extra }, encoding: 'utf8' + }); + assert.notEqual(run(['--file', 'mapping.json']).status, 0); + writeFileSync(jar, 'fixture'); + const result = run(['-p', '2', '--', '--file', 'path with spaces.json', '--token', 'fake-token', + '--cdc-sink-parallelism', '2']); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout, ''); + assert.equal(result.stderr, ''); + assert.deepEqual(readFileSync(argsFile, 'utf8').split('\0').slice(0, -1), + ['run', '-p', '2', '-c', 'org.apache.hugegraph.loader.flink.HugeGraphFlinkCDCLoader', jar, + '--file', 'path with spaces.json', '--token', 'fake-token', '--cdc-sink-parallelism', '2']); + const direct = run(['--file', 'mapping.json', '--graph', 'fixture']); + assert.equal(direct.status, 0, direct.stderr); + assert.deepEqual(readFileSync(argsFile, 'utf8').split('\0').slice(0, -1), + ['run', '-c', 'org.apache.hugegraph.loader.flink.HugeGraphFlinkCDCLoader', jar, + '--file', 'mapping.json', '--graph', 'fixture']); + assert.notEqual(run([], { FLINK_HOME: '' }).status, 0); + writeFileSync(join(root, 'loader/lib/apache-hugegraph-loader-extra-shaded.jar'), 'fixture'); + assert.notEqual(run([]).status, 0); +}); + +test('ordinary Loader excludes the shared engine job and keeps Kafka clients', t => { + const root = realpathSync(mkdtempSync(join(tmpdir(), 'loader-classpath-'))); + t.after(() => rmSync(root, { recursive: true, force: true })); + mkdirSync(join(root, 'loader/bin'), { recursive: true }); + mkdirSync(join(root, 'loader/lib')); + mkdirSync(join(root, 'java/bin'), { recursive: true }); + const script = join(root, 'loader/bin/hugegraph-loader.sh'); + copyFileSync(resolve(__dirname, '../../../assembly/static/bin/hugegraph-loader.sh'), script); + const argsFile = join(root, 'args'); + writeFileSync(join(root, 'java/bin/java'), '#!/bin/bash\n' + + 'if [[ "$1" == "-version" ]]; then echo \'openjdk version "17.0.1"\' >&2; exit 0; fi\n' + + 'printf "%s\\0" "$@" > "$PROBE_ARGS"\n', { mode: 0o755 }); + for (const name of ['apache-hugegraph-loader-1.8.0.jar', 'apache-hugegraph-loader-1.8.0-shaded.jar', + 'kafka-clients-3.2.0.jar', 'zstd-jni-1.5.0-4.jar', 'log4j-slf4j-impl-2.25.5.jar', 'custom-shaded.jar']) { + writeFileSync(join(root, 'loader/lib', name), 'fixture'); + } + const result = spawnSync('bash', [script, '--file', 'mapping.json'], { + env: { ...process.env, JAVA_HOME: join(root, 'java'), PROBE_ARGS: argsFile, + CLASSPATH: '', JVM_OPTS: '' }, encoding: 'utf8' + }); + assert.equal(result.status, 0, result.stderr); + const args = readFileSync(argsFile, 'utf8').split('\0').slice(0, -1); + const classpath = args[args.indexOf('-cp') + 1].split(':'); + assert(!classpath.includes(join(root, 'loader/lib/apache-hugegraph-loader-1.8.0-shaded.jar'))); + for (const name of ['apache-hugegraph-loader-1.8.0.jar', 'kafka-clients-3.2.0.jar', + 'zstd-jni-1.5.0-4.jar', 'log4j-slf4j-impl-2.25.5.jar', 'custom-shaded.jar']) { + assert(classpath.includes(join(root, 'loader/lib', name)), name); + } +}); diff --git a/pom.xml b/pom.xml index fb949e21f..793b3cbc9 100644 --- a/pom.xml +++ b/pom.xml @@ -113,8 +113,8 @@ 3.15.0 3.5.6 0.8.15 - 1.13.5 - 2.2.1 + 2.2.1 + 3.6.0-2.2 3.2.2 provided provided From 59aacb28f69f218c72f9a2c7463dffb2426170a2 Mon Sep 17 00:00:00 2001 From: imbajin Date: Thu, 8 Oct 2026 22:30:30 +0800 Subject: [PATCH 2/5] fix(ci): verify current ordinary PR merges - recover stale event bases using the current PR merge - verify canonical branch, merge parents and source head - preserve native stack checks and final gate freshness - cover ordinary snapshots and rejected stale inputs --- .github/scripts/ci-policy.py | 40 ++++++++++++--- .github/scripts/test_ci_policy.py | 83 +++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 7 deletions(-) diff --git a/.github/scripts/ci-policy.py b/.github/scripts/ci-policy.py index 193efcc41..46e6b60c0 100644 --- a/.github/scripts/ci-policy.py +++ b/.github/scripts/ci-policy.py @@ -279,14 +279,38 @@ def require_current_pr(plan, fetch): or live["head"]["ref"] != plan["branch"] or live["base"]["repo"]["full_name"] != plan["repository"]): raise StaleInputError("PR inputs changed; refresh the branch and start a new PR run") - if "stack" in plan: - # Native stacks can leave REST base.sha at an older snapshot. The actual - # branch ref, rather than that snapshot, identifies the tested base. + if "baseBranch" in plan: + # Event and REST base.sha can be older snapshots. The actual branch + # ref, rather than those snapshots, identifies the tested base. ref = fetch(f"repos/{plan['repository']}/git/ref/heads/{quote(plan['baseBranch'], safe='')}") - if (live.get("stack") != plan["stack"] or live["base"].get("ref") != plan["baseBranch"] + if (live.get("stack") != plan.get("stack") or live["base"].get("ref") != plan["baseBranch"] or ref.get("object", {}).get("sha") != plan["base"] or live.get("merge_commit_sha") != plan["testedMergeSHA"]): - raise StaleInputError("PR stack or canonical base changed; start a new PR run") + raise StaleInputError("PR merge or canonical base changed; start a new PR run") + + +def record_current_merge(plan, event_pr, parents, fetch): + """Recover an older event base only from GitHub's current merge identity.""" + try: + live = fetch(f"repos/{plan['repository']}/pulls/{plan['pr']}") + if live.get("stack") is not None: + record_stack_merge(plan, event_pr, parents, fetch) + return + branch = live["base"]["ref"] + if not branch or event_pr["base"]["ref"] != branch: + raise StaleInputError("PR target branch changed; start a new PR run") + ref = fetch(f"repos/{plan['repository']}/git/ref/heads/{quote(branch, safe='')}") + canonical = ref["object"]["sha"] + if (event_pr.get("stack") is not None + or ref["object"].get("type") != "commit" or not canonical + or live.get("merge_commit_sha") != plan["testedMergeSHA"] + or parents != [canonical, plan["head"]] + or git("merge-base", plan["base"], canonical) != plan["base"]): + raise StaleInputError("checkout lacks a valid current PR merge identity") + plan.update(base=canonical, baseBranch=branch) + require_current_pr(plan, fetch) + except (ValueError, KeyError, TypeError, AttributeError, subprocess.SubprocessError, OSError) as error: + raise StaleInputError("current PR merge identity could not be verified") from error def record_stack_merge(plan, event_pr, parents, fetch): @@ -343,7 +367,7 @@ def create_plan(project, event, repository, fetch=api, external_inputs=None): raise StaleInputError("PR event has an empty input identity") parents = git("show", "-s", "--format=%P", plan["testedMergeSHA"]).split() if parents != [plan["base"], plan["head"]]: - record_stack_merge(plan, pr, parents, fetch) + record_current_merge(plan, pr, parents, fetch) else: require_current_pr(plan, fetch) # head/base objects must exist locally; workflow fetches both before planning. @@ -424,8 +448,10 @@ def write_summary(plan, results=None): def result_report(plan, results): report = {key: plan[key] for key in ["schema", "repository", "project", "pr", "source", "branch", "base", "head", "testedMergeSHA", "externalInputs"]} + if "baseBranch" in plan: + report["baseBranch"] = plan["baseBranch"] if "stack" in plan: - report.update(baseBranch=plan["baseBranch"], stack=plan["stack"]) + report["stack"] = plan["stack"] report.update(runID=int(os.environ.get("GITHUB_RUN_ID", "0")), runAttempt=int(os.environ.get("GITHUB_RUN_ATTEMPT", "1")), executed=plan["expected"], results=results) diff --git a/.github/scripts/test_ci_policy.py b/.github/scripts/test_ci_policy.py index 77e22fd97..2f722f6bc 100644 --- a/.github/scripts/test_ci_policy.py +++ b/.github/scripts/test_ci_policy.py @@ -421,6 +421,89 @@ def git(*args): finally: os.chdir(old) + def test_regular_pr_recovers_stale_event_base_with_real_git(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + def git(*args): + return subprocess.check_output(["git", *args], cwd=root, text=True).strip() + git("init", "-q") + git("config", "user.email", "ci@example.invalid") + git("config", "user.name", "CI") + (root / "README.md").write_text("base") + (root / "hugegraph-hubble").mkdir() + (root / "hugegraph-hubble/README.md").write_text("packaged docs") + git("add", ".") + git("commit", "-qm", "base") + anchor = git("rev-parse", "HEAD") + git("checkout", "-qb", "feature") + (root / "hugegraph-loader").mkdir() + (root / "hugegraph-loader/A.java").write_text("source") + git("add", ".") + git("commit", "-qm", "feature") + head = git("rev-parse", "HEAD") + old_merge = git("commit-tree", git("rev-parse", "HEAD^{tree}"), + "-p", anchor, "-p", head, "-m", "old merge") + git("checkout", "-qb", "target", anchor) + (root / "README.md").write_text("advanced master") + git("commit", "-qam", "advance target") + canonical = git("rev-parse", "HEAD") + git("merge", "--no-ff", "-qm", "current merge", "feature") + merge = git("rev-parse", "HEAD") + live = self.live_pr() + live["head"]["sha"] = head + live["base"].update(sha=anchor, ref="target") + event = {"pull_request": json.loads(json.dumps(dict(live, number=7)))} + live.update(merge_commit_sha=merge) + ref = {"object": {"type": "commit", "sha": canonical}} + def fetch(path): + if path == "repos/apache/t/pulls/7": + return live + self.assertEqual("repos/apache/t/git/ref/heads/target", path) + return ref + old = os.getcwd() + try: + os.chdir(root) + # Neither event nor REST snapshot has stack metadata. The real + # checkout merges the current canonical target with the same head. + plan = policy.create_plan("toolchain", event, "apache/t", fetch) + self.assertEqual(canonical, plan["base"]) + self.assertEqual("target", plan["baseBranch"]) + self.assertNotIn("stack", plan) + self.assertEqual(["hubble", "loader"], plan["selected"]) + results = {suite: {"result": "success"} for suite in plan["expected"]} + results.update(plan={"result": "success"}, fixture={"result": "success"}, + **{"hubble-fixture": {"result": "success"}}) + report = policy.gate(plan, results, fetch) + self.assertEqual(canonical, report["base"]) + self.assertEqual("target", report["baseBranch"]) + for key, value in [("merge_commit_sha", old_merge), + ("state", "closed"), + ("head", dict(live["head"], sha=anchor)), + ("base", dict(live["base"], ref="other"))]: + original = live[key] + live[key] = value + with self.subTest(field=key), self.assertRaises(policy.StaleInputError): + policy.create_plan("toolchain", event, "apache/t", fetch) + with self.subTest(gate=key), self.assertRaises(policy.StaleInputError): + policy.gate(plan, results, fetch) + live[key] = original + ref["object"]["sha"] = anchor + with self.assertRaises(policy.StaleInputError): + policy.create_plan("toolchain", event, "apache/t", fetch) + with self.assertRaises(policy.StaleInputError): + policy.gate(plan, results, fetch) + ref["object"]["sha"] = canonical + # A rerun of an old checkout cannot claim the fresh merge. + git("checkout", "--detach", "-q", old_merge) + with self.assertRaises(policy.StaleInputError): + policy.gate(plan, results, lambda _: dict(live, head=dict(live["head"], sha=anchor))) + stale_event = json.loads(json.dumps(event)) + stale_event["pull_request"]["base"]["sha"] = canonical + with self.assertRaises(policy.StaleInputError): + policy.create_plan("toolchain", stale_event, "apache/t", fetch) + finally: + os.chdir(old) + def test_native_stack_canonical_base_and_gate_freshness(self): # Reproduce GitHub native stacks: REST base.sha can remain at the old # head, while the merge's first parent wraps the actual branch head. From 3cf693cddb3aaf438401b0221a13fa77296791d0 Mon Sep 17 00:00:00 2001 From: imbajin Date: Fri, 9 Oct 2026 00:30:41 +0800 Subject: [PATCH 3/5] fix(ci): follow verified upstream SDK versions - select floating consumer coordinates from the SDK manifest - preserve source, hash and local-install checks for packaging - retain the fixed candidate version outside compatibility CI - verify Maven version selection and separate CLI operations --- .github/scripts/test_candidate_sdk.py | 99 ++++++++++++++++++- .github/scripts/verify_candidate_image_sdk.py | 34 +++++-- .github/workflows/upstream-compatibility.yml | 7 ++ docs/ci.md | 2 + 4 files changed, 130 insertions(+), 12 deletions(-) diff --git a/.github/scripts/test_candidate_sdk.py b/.github/scripts/test_candidate_sdk.py index 1621defeb..8d93d298c 100644 --- a/.github/scripts/test_candidate_sdk.py +++ b/.github/scripts/test_candidate_sdk.py @@ -105,6 +105,53 @@ def run(*arguments): for path, original in original_poms.items(): self.assertEqual(original, path.read_bytes()) + @unittest.skipUnless(os.environ.get("CANDIDATE_SDK_MAVEN_TEST"), + "Set CANDIDATE_SDK_MAVEN_TEST for the consumer-version regression") + def test_maven_consumers_resolve_the_selected_sdk_version(self): + # A floating SDK changes coordinates; the consumer's fixed default must + # not silently resolve a different release from another repository. + import zipfile + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + group = "org.example.candidatesdk.g" + uuid.uuid4().hex + repository = Path.home() / ".m2/repository" / group.replace(".", "/") + self.addCleanup(shutil.rmtree, repository, True) + artifact = repository / "sdk/1.8.0" + artifact.mkdir(parents=True) + header = '4.0.0' + (artifact / "sdk-1.8.0.pom").write_text( + header + '' + group + 'sdk' + '1.8.0') + with zipfile.ZipFile(artifact / "sdk-1.8.0.jar", "w") as jar: + jar.writestr("sdk-source.txt", "selected upstream revision") + (artifact / "_remote.repositories").write_text("sdk-1.8.0.pom>=\nsdk-1.8.0.jar>=\n") + pom = root / "pom.xml" + pom.write_text(header + 'org.example.consumerconsumer' + '11.7.0' + '' + group + '' + 'sdk${hugegraph.version}' + '') + settings = root / "settings.xml" + settings.write_text("") + command = [os.environ["CANDIDATE_SDK_MAVEN_TEST"], "-o", "-ntp", "--settings", str(settings), + "-f", str(pom), "org.apache.maven.plugins:maven-dependency-plugin:3.1.1:build-classpath", + "-Dmdep.outputFile=" + str(root / "classpath")] + env = dict(os.environ, MAVEN_ARGS="-Dhugegraph.version=1.8.0") + # Warm only Maven's public plugin dependencies when this opt-in + # regression runs on a fresh machine; coordinate checks stay offline. + prepared = subprocess.run([arg for arg in command if arg != "-o"], env=env, + capture_output=True, text=True, timeout=90) + self.assertEqual(0, prepared.returncode, prepared.stdout + prepared.stderr) + (root / "classpath").unlink() + env["MAVEN_ARGS"] = "" + wrong = subprocess.run(command, env=env, capture_output=True, text=True, timeout=90) + self.assertNotEqual(0, wrong.returncode, wrong.stdout + wrong.stderr) + self.assertIn(group + ":sdk:jar:1.7.0", wrong.stdout + wrong.stderr) + env["MAVEN_ARGS"] = "-Dhugegraph.version=1.8.0" + selected = subprocess.run(command, env=env, capture_output=True, text=True, timeout=90) + self.assertEqual(0, selected.returncode, selected.stdout + selected.stderr) + self.assertEqual(str(artifact / "sdk-1.8.0.jar"), (root / "classpath").read_text().strip()) + class CandidateDistributionTest(unittest.TestCase): def setUp(self): @@ -113,7 +160,7 @@ def setUp(self): context.start() self.addCleanup(context.stop) - def fixture(self, root): + def fixture(self, root, revision="1.7.0"): repository = root / "m2" artifacts, modules = [], [] for source_pom in sorted(sdk.REQUIRED_MODULES): @@ -121,8 +168,8 @@ def fixture(self, root): packaging = "pom" if (source_pom.count("/") == 1 and artifact != "hugegraph-struct") or source_pom == "pom.xml" else "jar" files = [] for extension in (["pom", "jar"] if packaging == "jar" else ["pom"]): - filename = artifact + "-1.7.0." + extension - path = repository / "org/apache/hugegraph" / artifact / "1.7.0" / filename + filename = artifact + "-" + revision + "." + extension + path = repository / "org/apache/hugegraph" / artifact / revision / filename path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(filename.encode()) with (path.parent / "_remote.repositories").open("a") as origins: @@ -131,14 +178,56 @@ def fixture(self, root): files.append(relative) artifacts.append({"path": relative, "sha256": sdk.digest(path), "source_reactor_install": True}) modules.append({"group_id": "org.apache.hugegraph", "artifact_id": artifact, - "version": "1.7.0", "packaging": packaging, + "version": revision, "packaging": packaging, "source_pom": source_pom, "files": files}) manifest = {"repository": sdk.REPOSITORY, "commit": self.source_commit, - "source_revision": "1.7.0", "java_version": "17", + "source_revision": revision, "java_version": "17", "required_sdk_modules": modules, "artifacts": artifacts} self.write_manifest(repository, manifest) return repository, manifest + def test_floating_sdk_version_preserves_verified_coordinates_and_maven_arguments(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + repository, manifest = self.fixture(root, "1.8.0") + environment = root / "github-env" + # Fixed-baseline validation must still reject the newer revision. + with self.assertRaises(RuntimeError): + sdk.validate_sdk(repository) + original = "--settings settings.xml -Dmaven.repo.local=" + str(repository) + with patch.dict(os.environ, {"MAVEN_ARGS": original}): + sdk.configure_upstream(repository, environment) + lines = dict(line.split("=", 1) for line in environment.read_text().splitlines()) + self.assertEqual("1.8.0", lines["CANDIDATE_SDK_VERSION"]) + self.assertEqual(original + " -Dhugegraph.version=1.8.0", lines["MAVEN_ARGS"]) + with patch.dict(os.environ, lines): + sdk.validate_sdk(repository) + distribution = self.distribution(root, repository, manifest) + sdk.validate_distribution(repository, distribution, "loader") + # Hash checks must remain enforced for the floating version. + jar = repository / next(a["path"] for a in manifest["artifacts"] + if a["path"].endswith(".jar")) + jar.write_bytes(b"changed SDK") + with self.assertRaises(RuntimeError): + sdk.validate_sdk(repository) + with self.assertRaises(RuntimeError): + sdk.configure_upstream(repository, root / "invalid-env") + self.assertFalse((root / "invalid-env").exists()) + + def test_configuration_and_distribution_cli_options_are_separate_steps(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + repository, manifest = self.fixture(root, "1.8.0") + distribution = self.distribution(root, repository, manifest) + environment = root / "github-env" + result = subprocess.run(["python3", sdk.__file__, str(repository), + "--configure-upstream-env", str(environment), + "--distribution", str(distribution), "--module", "loader"], + capture_output=True, text=True, timeout=10) + self.assertEqual(2, result.returncode) + self.assertIn("cannot be combined", result.stderr) + self.assertFalse(environment.exists()) + def write_manifest(self, repository, manifest): (repository / "candidate-sdk-manifest.json").write_text(json.dumps(manifest)) diff --git a/.github/scripts/verify_candidate_image_sdk.py b/.github/scripts/verify_candidate_image_sdk.py index 291ae46d6..aa7cb52ee 100644 --- a/.github/scripts/verify_candidate_image_sdk.py +++ b/.github/scripts/verify_candidate_image_sdk.py @@ -61,15 +61,15 @@ def source_commit(): return expected -def validate_sdk(repository): +def validate_sdk(repository, expected_revision=None): try: - return _validate_sdk(Path(repository)) + return _validate_sdk(Path(repository), expected_revision) except (OSError, ValueError, KeyError, TypeError, AttributeError) as error: raise RuntimeError("Invalid candidate SDK manifest or artifact metadata (" + type(error).__name__ + ")") from None -def _validate_sdk(repository): +def _validate_sdk(repository, expected_revision=None): repository = Path(repository) manifest = json.loads((repository / "candidate-sdk-manifest.json").read_text()) if (manifest["repository"], manifest["commit"]) != (REPOSITORY, source_commit()): @@ -77,7 +77,10 @@ def _validate_sdk(repository): java_version = manifest["java_version"] if not isinstance(java_version, str) or not java_version.isdecimal(): raise ValueError("Invalid Java version format") - if manifest["source_revision"] != "1.7.0" or java_version != "17": + revision = expected_revision or os.environ.get("CANDIDATE_SDK_VERSION", "1.7.0") + if not isinstance(revision, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.+-]*", revision): + raise ValueError("Invalid candidate SDK version") + if manifest["source_revision"] != revision or java_version != "17": raise RuntimeError("SDK revision or Java version does not match the candidate contract") modules = manifest["required_sdk_modules"] if len(modules) != len(REQUIRED_MODULES) or {m["source_pom"] for m in modules} != REQUIRED_MODULES: @@ -87,7 +90,7 @@ def _validate_sdk(repository): raise RuntimeError("Duplicate SDK artifact paths") required_files = set() for module in modules: - if module["group_id"] != "org.apache.hugegraph" or module["version"] != "1.7.0": + if module["group_id"] != "org.apache.hugegraph" or module["version"] != revision: raise RuntimeError("Unexpected required SDK coordinates") artifact = module["artifact_id"] source_pom = module["source_pom"] @@ -97,7 +100,7 @@ def _validate_sdk(repository): if artifact != expected_artifact or module["packaging"] != ("pom" if parent else "jar"): raise RuntimeError("Required SDK module does not match its source coordinates") extensions = {"pom", "jar"} if module["packaging"] == "jar" else {"pom"} - expected = {f"org/apache/hugegraph/{artifact}/1.7.0/{artifact}-1.7.0.{ext}" + expected = {f"org/apache/hugegraph/{artifact}/{revision}/{artifact}-{revision}.{ext}" for ext in extensions} if set(module["files"]) != expected: raise RuntimeError("Required SDK module has missing or unexpected files") @@ -160,16 +163,33 @@ def validate_distribution(repository, directory, module): return actual +def configure_upstream(repository, environment): + """Select consumer coordinates only after verifying this run's SDK artifacts.""" + repository = Path(repository) + revision = json.loads((repository / "candidate-sdk-manifest.json").read_text())["source_revision"] + manifest, jars = validate_sdk(repository, revision) + with Path(environment).open("a") as stream: + stream.write(f"CANDIDATE_SDK_VERSION={revision}\n") + stream.write(f"MAVEN_ARGS={os.environ.get('MAVEN_ARGS', '')} -Dhugegraph.version={revision}\n") + return manifest, jars + + if __name__ == "__main__": parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("repository", type=Path) parser.add_argument("--distribution", type=Path) + parser.add_argument("--configure-upstream-env", type=Path) parser.add_argument("--module", choices=("loader", "tools", "hubble")) args = parser.parse_args() if bool(args.distribution) != bool(args.module): parser.error("--distribution and --module must be used together") + if args.configure_upstream_env and args.distribution: + parser.error("--configure-upstream-env cannot be combined with --distribution") try: - manifest, jars = validate_sdk(args.repository) + if args.configure_upstream_env: + manifest, jars = configure_upstream(args.repository, args.configure_upstream_env) + else: + manifest, jars = validate_sdk(args.repository) if args.distribution: validate_distribution(args.repository, args.distribution, args.module) except RuntimeError as error: diff --git a/.github/workflows/upstream-compatibility.yml b/.github/workflows/upstream-compatibility.yml index 19fdee8a6..e2e95d360 100644 --- a/.github/workflows/upstream-compatibility.yml +++ b/.github/workflows/upstream-compatibility.yml @@ -53,6 +53,13 @@ jobs: ref: ${{ steps.upstream.outputs.commit_id }} expected-commit: ${{ steps.upstream.outputs.commit_id }} + - name: Select verified SDK version for all consumer builds and tests + env: + CANDIDATE_REPO: ${{ steps.sdk.outputs.maven-repo }} + run: >- + python3 .github/scripts/verify_candidate_image_sdk.py "$CANDIDATE_REPO" + --configure-upstream-env "$GITHUB_ENV" + - name: Compile core consumers run: | mvn install -pl hugegraph-client,hugegraph-loader,hugegraph-tools,hugegraph-spark-connector -am -DskipTests -Dmaven.javadoc.skip=true -ntp diff --git a/docs/ci.md b/docs/ci.md index 1bc6897e0..c3057ebc0 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -70,3 +70,5 @@ actionlint ``` Upstream compatibility runs daily and on demand against the latest ASF master, resolved once to an immutable commit for each run. It reuses the SDK build, core compilation, Client HTTP tests, Loader and Hubble backend unit tests, and Tools functional tests; Hadoop, Spark/Flink engine matrices and frontend browser tests remain in module CI. This independent check does not change the locked Module validation baseline. + +The floating upstream check selects `hugegraph.version` from the verified SDK manifest for every consumer build and test. The same version is passed to SDK packaging verification through `CANDIDATE_SDK_VERSION`; provenance, local-install and artifact-hash checks remain enforced. This lets Server master advance its Maven version without accidentally testing a released SDK. Fixed-baseline jobs retain their default SDK version. From fc89ada96ed572817790d5ebdc02004910b3221e Mon Sep 17 00:00:00 2001 From: imbajin <17706099+imbajin@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:24:25 +0800 Subject: [PATCH 4/5] fix(loader): preserve INT96 timestamps without noon offset --- .../hugegraph/loader/util/ParquetUtil.java | 4 +- .../loader/test/unit/ParquetUtilTest.java | 74 +++++++++++++++++++ .../loader/test/unit/UnitTestSuite.java | 1 + 3 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/ParquetUtilTest.java diff --git a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/ParquetUtil.java b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/ParquetUtil.java index cafcbefbf..f084fc1c7 100644 --- a/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/ParquetUtil.java +++ b/hugegraph-loader/src/main/java/org/apache/hugegraph/loader/util/ParquetUtil.java @@ -79,7 +79,7 @@ private static Date dateFromInt96(Binary value) { julianDay += int96Bytes[index] & 0xFF; } - // Find nanos since midday (since Julian days start at midday) + // Parquet INT96 stores nanoseconds since midnight. long nanos = 0; // Continue from the index we got to while (index > 0) { @@ -90,7 +90,7 @@ private static Date dateFromInt96(Binary value) { LocalDateTime timestamp = LocalDate.MIN.with(JulianFields.JULIAN_DAY, julianDay) - .atTime(LocalTime.NOON) + .atTime(LocalTime.MIDNIGHT) .plusNanos(nanos); return Date.from(timestamp.atZone(ZoneId.systemDefault()).toInstant()); } diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/ParquetUtilTest.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/ParquetUtilTest.java new file mode 100644 index 000000000..daf3b8c06 --- /dev/null +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/ParquetUtilTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.loader.test.unit; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneId; +import java.util.Date; + +import org.apache.hugegraph.loader.util.ParquetUtil; +import org.apache.hugegraph.testutil.Assert; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.io.api.Binary; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; +import org.junit.Test; + +public class ParquetUtilTest { + + private static final long JULIAN_DAY_AT_UNIX_EPOCH = 2440588L; + + @Test + public void testInt96Midnight() { + assertTimestamp(LocalDate.of(2019, 12, 9), LocalTime.MIDNIGHT); + } + + @Test + public void testInt96FractionalSecondsAcrossDayBoundary() { + assertTimestamp(LocalDate.of(2020, 2, 29), + LocalTime.of(23, 59, 59, 123456789)); + } + + @Test + public void testInt96BeforeUnixEpoch() { + assertTimestamp(LocalDate.of(1969, 12, 31), LocalTime.MIDNIGHT); + } + + private static void assertTimestamp(LocalDate day, LocalTime time) { + MessageType schema = MessageTypeParser.parseMessageType( + "message timestamp { required int96 value; }"); + Group group = new SimpleGroupFactory(schema).newGroup(); + // INT96 stores nanoseconds since midnight followed by the Julian day. + int julianDay = Math.toIntExact(day.toEpochDay() + + JULIAN_DAY_AT_UNIX_EPOCH); + byte[] encoded = ByteBuffer.allocate(12).order(ByteOrder.LITTLE_ENDIAN) + .putLong(time.toNanoOfDay()) + .putInt(julianDay).array(); + group.add("value", Binary.fromConstantByteArray(encoded)); + + Object actual = ParquetUtil.convertObject(group, 0); + Assert.assertTrue(actual instanceof Date); + long expected = day.atTime(time).atZone(ZoneId.systemDefault()) + .toInstant().toEpochMilli(); + Assert.assertEquals(expected, ((Date) actual).getTime()); + } +} diff --git a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/UnitTestSuite.java b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/UnitTestSuite.java index d1192ba07..ea2528590 100644 --- a/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/UnitTestSuite.java +++ b/hugegraph-loader/src/test/java/org/apache/hugegraph/loader/test/unit/UnitTestSuite.java @@ -32,6 +32,7 @@ SparkPartitionWriterTest.class, SparkRowTest.class, DateUtilTest.class, + ParquetUtilTest.class, MappingConverterTest.class, LoadProgressTest.class, RangesTimerTest.class, From 5255fd8650b7ca45270fb39fefa6c02e4bd9433c Mon Sep 17 00:00:00 2001 From: imbajin Date: Fri, 9 Oct 2026 10:18:19 +0800 Subject: [PATCH 5/5] fix(loader): align launcher regression contracts - check missing shared options through the Spark launcher - remove the replaced legacy CDC parser assertion - verify native Flink exit status propagation --- .../src/test/python/test_spark_launcher.py | 20 +++++-------------- .../src/test/scripts/flink-launcher.test.cjs | 6 +++++- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/hugegraph-loader/src/test/python/test_spark_launcher.py b/hugegraph-loader/src/test/python/test_spark_launcher.py index 94ccdba29..570fa3592 100644 --- a/hugegraph-loader/src/test/python/test_spark_launcher.py +++ b/hugegraph-loader/src/test/python/test_spark_launcher.py @@ -373,22 +373,12 @@ def test_shared_legacy_string_outputs_remain_available(self): result.stdout.splitlines()) def test_missing_value_fails_before_submit(self): - result = self.run_launcher("--password") - self.assertEqual(2, result.returncode) - self.assertIn("Missing value for --password", result.stderr) - self.assertFalse(self.argv.exists()) - - def test_legacy_cdc_missing_value_stops_before_command_construction(self): - source = Path(__file__).resolve().parents[3] / "assembly/static/bin" - launcher = self.app / "bin/hugegraph-flinkcdc-loader.sh" - shutil.copyfile(source / launcher.name, launcher) for option in ("--password", "--file", "--batch-size"): - result = subprocess.run(["bash", str(launcher), option], - text=True, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, timeout=10) - self.assertEqual(2, result.returncode, result.stderr) - self.assertIn("Missing value for " + option, result.stderr) - self.assertEqual("", result.stdout) + with self.subTest(option=option): + result = self.run_launcher(option) + self.assertEqual(2, result.returncode) + self.assertIn("Missing value for " + option, result.stderr) + self.assertFalse(self.argv.exists()) def test_help_does_not_consume_an_engine_parameter(self): result = self.run_launcher("--help", "--master", "local[1]") diff --git a/hugegraph-loader/src/test/scripts/flink-launcher.test.cjs b/hugegraph-loader/src/test/scripts/flink-launcher.test.cjs index c4ab07a58..cfa9cccbe 100644 --- a/hugegraph-loader/src/test/scripts/flink-launcher.test.cjs +++ b/hugegraph-loader/src/test/scripts/flink-launcher.test.cjs @@ -31,7 +31,8 @@ test('CDC launcher keeps argument boundaries and does not expose arguments', t = const script = join(root, 'loader/bin/hugegraph-flinkcdc-loader.sh'); copyFileSync(resolve(__dirname, '../../../assembly/static/bin/hugegraph-flinkcdc-loader.sh'), script); const argsFile = join(root, 'args'); - writeFileSync(join(root, 'flink/bin/flink'), '#!/bin/bash\nprintf "%s\\0" "$@" > "$PROBE_ARGS"\n', { mode: 0o755 }); + writeFileSync(join(root, 'flink/bin/flink'), '#!/bin/bash\nprintf "%s\\0" "$@" > "$PROBE_ARGS"\n' + + 'exit "${PROBE_EXIT_CODE:-0}"\n', { mode: 0o755 }); const jar = join(root, 'loader/lib/apache-hugegraph-loader-1.8.0-shaded.jar'); const run = (args, extra = {}) => spawnSync('bash', [script, ...args], { env: { ...process.env, FLINK_HOME: join(root, 'flink'), PROBE_ARGS: argsFile, ...extra }, encoding: 'utf8' @@ -51,6 +52,9 @@ test('CDC launcher keeps argument boundaries and does not expose arguments', t = assert.deepEqual(readFileSync(argsFile, 'utf8').split('\0').slice(0, -1), ['run', '-c', 'org.apache.hugegraph.loader.flink.HugeGraphFlinkCDCLoader', jar, '--file', 'mapping.json', '--graph', 'fixture']); + const rejected = run(['--file'], { PROBE_EXIT_CODE: '2' }); + assert.equal(rejected.status, 2, rejected.stderr); + assert.equal(rejected.stdout, ''); assert.notEqual(run([], { FLINK_HOME: '' }).status, 0); writeFileSync(join(root, 'loader/lib/apache-hugegraph-loader-extra-shaded.jar'), 'fixture'); assert.notEqual(run([]).status, 0);