feat: run new test concept everywhere

ytausch · ytausch · commit de9529882066 · 2025-03-27T19:21:06.000+01:00
diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml
@@ -17,271 +17,64 @@ defaults:
     shell: bash -leo pipefail {0}
 
 jobs:
-  setup-repositories:
-    name: Set up Integration Tests
+  integration-tests:
+    name: Run Integration Tests
     # if triggered by pull_request, only run on non-fork PRs (secrets access needed)
     # Nevertheless, this check is always run in the merge queue.
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
     runs-on: ubuntu-latest
-    outputs:
-      scenario_ids: ${{ steps.setup_repositories.outputs.scenario_ids }}
     steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
+
       - name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           path: cf-scripts
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3.8.0
+          submodules: 'true'
 
       - name: Build Docker Image
-        uses: docker/build-push-action@b32b51a8eda65d6793cd0494a773d4f6bcef32dc # v6.11.0
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
         with:
           context: cf-scripts
           push: false
           load: true
           tags: conda-forge-tick:test
           cache-from: type=gha
           cache-to: type=gha,mode=max
-          outputs: type=docker,dest=/tmp/image.tar
 
-      - name: Upload Docker Image
-        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+      - name: Setup micromamba
+        uses: mamba-org/setup-micromamba@0dea6379afdaffa5d528b3d1dabc45da37f443fc # v2.0.4
         with:
-          name: conda-forge-tick
-          path: /tmp/image.tar
-
-      - name: setup-micromamba
-        uses: mamba-org/setup-micromamba@4b9113af4fba0e9e1124b252dd6497a419e7396d # v1.11.0
-        with:
-          environment-file: cf-scripts/conda-lock.yml
+          environment-file: conda-lock.yml
           environment-name: cf-scripts
-          condarc-file: cf-scripts/autotick-bot/condarc
+          condarc-file: autotick-bot/condarc
 
       - name: Run pip install
         working-directory: cf-scripts
         run: |
           pip install --no-deps --no-build-isolation -e .
 
-      - name: Set up Integration Test Repositories
-        id: setup_repositories
+      - name: Run mitmproxy certificate setup wizard
         working-directory: cf-scripts
-        # note: this writes `scenario_ids` to GITHUB_OUTPUT
-        run: python -m tests_integration.setup_repositories
-        env:
-          TEST_SETUP_TOKEN: ${{ secrets.GH_TOKEN_STAGING_BOT_USER }}
-
-  run-test-scenario:
-    name: Run Scenarios
-    # if triggered by pull_request, only run on non-fork PRs (secrets access needed)
-    # Nevertheless, this check is always run in the merge queue.
-    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
-    runs-on: ubuntu-latest
-    needs: setup-repositories
-    strategy:
-      matrix:
-        scenario_id: ${{ fromJson(needs.setup-repositories.outputs.scenario_ids) }}
-      max-parallel: 1
-      fail-fast: false
-    steps:
-      - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          path: cf-scripts
-          submodules: 'true'
-
-      - name: Download Docker Image
-        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
-        with:
-          name: conda-forge-tick
-          path: /tmp
-
-      - name: setup-micromamba
-        uses: mamba-org/setup-micromamba@4b9113af4fba0e9e1124b252dd6497a419e7396d # v1.11.0
-        with:
-          environment-file: cf-scripts/conda-lock.yml
-          environment-name: cf-scripts
-          condarc-file: cf-scripts/autotick-bot/condarc
-
-      - name: Install Bot Code (no graph clone)
-        # this is the first time, so we clean disk space, but don't clone the graph
-        run: source cf-scripts/autotick-bot/install_bot_code.sh --no-pull-container --no-clone-graph
-
-      - name: Load Docker Image
-        run: |
-          docker load --input /tmp/image.tar
-          docker image ls -a
-
-      - name: Prepare Scenario
-        working-directory: cf-scripts
-        run: python -m tests_integration.step_prepare  # this creates the cf-graph repository
-        env:
-          SCENARIO_ID: ${{ matrix.scenario_id }}
-          TEST_SETUP_TOKEN: ${{ secrets.GH_TOKEN_STAGING_BOT_USER }}
-
-      - name: Install Bot Code
         run: |
-          source cf-scripts/autotick-bot/install_bot_code.sh --no-pull-container --no-clean-disk-space
-
-      - name: Start HTTP Proxy
-        working-directory: cf-scripts
-        run: |
-          ./tests_integration/mock_proxy_start.sh &
-          sleep 10
-          # Install CA Certificate
-          sudo wget -e use_proxy=yes -e http_proxy=127.0.0.1:8080 -O /usr/local/share/ca-certificates/mitmproxy.crt \
-            http://mitm.it/cert/pem
+          # place a script in the mitmproxy directory that will be run by the setup wizard
+          # to trust the mitmproxy certificate
+          cat <<EOF "${MITMPROXY_WIZARD_HEADLESS_TRUST_SCRIPT}"
+          #!/usr/bin/env bash
+          set -euo pipefail
+          sudo cp "$1" /usr/local/share/ca-certificates/mitmproxy.crt
           sudo update-ca-certificates
-        env:
-          SCENARIO_ID: ${{ matrix.scenario_id }}
-          PYTHONPATH: ${{ github.workspace }}/cf-scripts
+          EOF
 
-      - name: "[Test] Gather all Feedstocks"
-        working-directory: cf-graph
-        run: |
-          # don't use proxy for git CLI operations (applies to all further steps)
-          git config --global http."https://github.com".proxy ""
-          conda-forge-tick --debug gather-all-feedstocks
-
-      - name: "[Test] Deploy to GitHub"
-        working-directory: cf-graph
-        run: conda-forge-tick --debug deploy-to-github
-
-      - name: Reinstall Bot Code
-        run: |
-          source cf-scripts/tests_integration/clear_runner.sh
-          source cf-scripts/autotick-bot/install_bot_code.sh --no-clean-disk-space --no-pull-container
-
-      - name: "[Test] Make Graph (Nodes and Edges)"
-        working-directory: cf-graph
-        run: conda-forge-tick --debug make-graph --update-nodes-and-edges
-
-      - name: "[Test] Deploy to GitHub"
-        working-directory: cf-graph
-        run: conda-forge-tick --debug deploy-to-github
-
-      - name: Reinstall Bot Code
-        run: |
-          source cf-scripts/tests_integration/clear_runner.sh
-          source cf-scripts/autotick-bot/install_bot_code.sh --no-clean-disk-space --no-pull-container
-
-      - name: "[Test] Make Graph (Node Attributes)"
-        working-directory: cf-graph
-        run: conda-forge-tick --debug make-graph
-        env:
-          CF_TICK_FRAC_MAKE_GRAPH: "1.0"
-
-      - name: "[Test] Deploy to GitHub"
-        working-directory: cf-graph
-        run: conda-forge-tick --debug deploy-to-github
-
-      - name: Reinstall Bot Code
-        run: |
-          source cf-scripts/tests_integration/clear_runner.sh
-          source cf-scripts/autotick-bot/install_bot_code.sh --no-clean-disk-space --no-pull-container
-
-      - name: "[Test] update-upstream-versions"
-        working-directory: cf-graph
-        run: conda-forge-tick --debug update-upstream-versions
-        env:
-          http_proxy: http://127.0.0.1:8080
-          https_proxy: http://127.0.0.1:8080
-          SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
-          REQUESTS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
-          CF_FEEDSTOCK_OPS_CONTAINER_PROXY_MODE: true
-          CF_FEEDSTOCK_OPS_PROXY_IN_CONTAINER: http://172.17.0.1:8080
-
-      - name: "[Test] Deploy to GitHub"
-        working-directory: cf-graph
-        run: conda-forge-tick --debug deploy-to-github
-
-      - name: Reinstall Bot Code
-        run: |
-          source cf-scripts/tests_integration/clear_runner.sh
-          source cf-scripts/autotick-bot/install_bot_code.sh --no-clean-disk-space --no-pull-container
-
-      - name: "[Test] make-migrators (1/2)"
-        working-directory: cf-graph
-        run: conda-forge-tick --debug make-migrators
-        env:
-          http_proxy: http://127.0.0.1:8080
-          https_proxy: http://127.0.0.1:8080
-          SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
-          REQUESTS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
-          CF_FEEDSTOCK_OPS_CONTAINER_PROXY_MODE: true
-          CF_FEEDSTOCK_OPS_PROXY_IN_CONTAINER: http://172.17.0.1:8080
-
-      - name: "[Test] Deploy to GitHub"
-        working-directory: cf-graph
-        run: conda-forge-tick --debug deploy-to-github
-
-      - name: Reinstall Bot Code
-        run: |
-          source cf-scripts/tests_integration/clear_runner.sh
-          source cf-scripts/autotick-bot/install_bot_code.sh --no-clean-disk-space --no-pull-container
-
-      - name: "[Test] auto-tick (1/2)"
-        working-directory: cf-graph
-        run: conda-forge-tick --debug auto-tick
-        env:
-          http_proxy: http://127.0.0.1:8080
-          https_proxy: http://127.0.0.1:8080
-          SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
-          REQUESTS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
-          CF_FEEDSTOCK_OPS_CONTAINER_PROXY_MODE: true
-          CF_FEEDSTOCK_OPS_PROXY_IN_CONTAINER: http://172.17.0.1:8080
-
-      - name: "[Test] Deploy to GitHub"
-        working-directory: cf-graph
-        run: conda-forge-tick --debug deploy-to-github
-
-      - name: Reinstall Bot Code
-        run: |
-          source cf-scripts/tests_integration/clear_runner.sh
-          source cf-scripts/autotick-bot/install_bot_code.sh --no-clean-disk-space --no-pull-container
-
-      - name: "[Test] make-migrators (2/2)"
-        working-directory: cf-graph
-        run: conda-forge-tick --debug make-migrators
+          ./tests_integration/mitmproxy_setup_wizard.sh
         env:
-          http_proxy: http://127.0.0.1:8080
-          https_proxy: http://127.0.0.1:8080
-          SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
-          REQUESTS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
-          CF_FEEDSTOCK_OPS_CONTAINER_PROXY_MODE: true
-          CF_FEEDSTOCK_OPS_PROXY_IN_CONTAINER: http://172.17.0.1:8080
-
-      - name: "[Test] Deploy to GitHub"
-        working-directory: cf-graph
-        run: conda-forge-tick --debug deploy-to-github
-
-      - name: Reinstall Bot Code
-        run: |
-          source cf-scripts/tests_integration/clear_runner.sh
-          source cf-scripts/autotick-bot/install_bot_code.sh --no-clean-disk-space --no-pull-container
+          MITMPROXY_WIZARD_HEADLESS: true
+          MITMPROXY_WIZARD_HEADLESS_TRUST_SCRIPT: ./tests_integration/.mitmproxy/mitmproxy_trust_script.sh
 
-      - name: "[Test] auto-tick (2/2)"
-        working-directory: cf-graph
-        run: conda-forge-tick --debug auto-tick
-        env:
-          http_proxy: http://127.0.0.1:8080
-          https_proxy: http://127.0.0.1:8080
-          SSL_CERT_FILE: /etc/ssl/certs/ca-certificates.crt
-          REQUESTS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
-          CF_FEEDSTOCK_OPS_CONTAINER_PROXY_MODE: true
-          CF_FEEDSTOCK_OPS_PROXY_IN_CONTAINER: http://172.17.0.1:8080
-
-      - name: "[Test] Deploy to GitHub"
-        working-directory: cf-graph
-        run: conda-forge-tick --debug deploy-to-github
-
-      - name: Validate Scenario
+      - name: Run Integration Tests with pytest
         working-directory: cf-scripts
-        run: python -m tests_integration.step_validate
+        run: pytest -s -v tests_integration
         env:
-          SCENARIO_ID: ${{ matrix.scenario_id }}
+          BOT_TOKEN: ${{ secrets.GH_TOKEN_STAGING_BOT_USER }}
           TEST_SETUP_TOKEN: ${{ secrets.GH_TOKEN_STAGING_BOT_USER }}
-
-      - name: Print Proxy Logs
-        if: always()
-        run: cat /tmp/mitmproxy.log
diff --git a/tests_integration/mitmproxy_setup_wizard.sh b/tests_integration/mitmproxy_setup_wizard.sh
@@ -1,7 +1,9 @@
+#!/usr/bin/env bash
+
 set -euo pipefail
 
 echo "=== mitmproxy certificates setup wizard ==="
-echo "Use this shell script to setup the mitmproxy certificates for the integration tests on your local machine."
+echo "Use this shell script to setup the mitmproxy certificates for the integration tests on your machine."
 
 # we could also add openssl to the conda environment, but this should be available on most systems
 if ! command -v openssl &> /dev/null; then
@@ -11,26 +13,47 @@ fi
 
 script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 mitmproxy_dir="${script_dir}/.mitmproxy"
-
 # the mitmproxy_dir should already exist
 cd "${mitmproxy_dir}"
 
-echo "Press enter to generate the mitmproxy certificates."
-read -r
+# Headless Mode is used in GitHub Actions only
+headless_mode="${MITMPROXY_WIZARD_HEADLESS:-false}"
+
+if [ "${headless_mode}" = "true" ]; then
+    echo "Running in headless mode."
+    echo "The mitmproxy certificates will be generated in the directory: ${mitmproxy_dir}"
+else
+    echo "The mitmproxy certificates will be generated in the directory: ${mitmproxy_dir}"
+    echo "Press enter to continue or Ctrl+C to cancel."
+    read -r
+fi
 
 openssl genrsa -out mitmproxy-ca.key 4096
 openssl req -x509 -new -nodes -key mitmproxy-ca.key -sha256 -days 365 -out mitmproxy-ca.crt -addext keyUsage=critical,keyCertSign -subj "/C=US/ST=cf-scripts/L=cf-scripts/O=cf-scripts/OU=cf-scripts/CN=cf-scripts"
 cat mitmproxy-ca.key mitmproxy-ca.crt > mitmproxy-ca.pem
 
+# path to a script that will be executed after the certificates have been generated
+# the script should add the mitmproxy-ca.pem certificate to the system's trust store
+# the first argument is the path to the mitmproxy-ca.pem certificate
+headless_mode_trust_script="${MITMPROXY_WIZARD_HEADLESS_TRUST_SCRIPT})"
+
 echo "The mitmproxy certificates have been generated successfully."
 echo "The root certificate will be valid for 365 days."
-echo "You now need to trust the mitmproxy-ca.pem certificate in your system's trust store."
-echo "The exact process depends on your operating system."
-echo "On MacOS, drag and drop the mitmproxy-ca.pem file into the Keychain Access app while having the 'Login' keychain selected."
-echo "Then, double-click the certificate in the keychain and set ‘Always Trust‘ in the ‘Trust‘ section."
-echo "The certificate is located at: ${mitmproxy_dir}/mitmproxy-ca.pem"
-echo "After you're done, press enter to continue."
-read -r
+
+mitmproxy_ca_pem_file="${mitmproxy_dir}/mitmproxy-ca.pem"
+
+if [ "${headless_mode}" = "true" ]; then
+    echo "Executing the headless mode trust script..."
+    bash "${headless_mode_trust_script}" "${mitmproxy_ca_pem_file}"
+else
+    echo "You now need to trust the mitmproxy-ca.pem certificate in your system's trust store."
+    echo "The exact process depends on your operating system."
+    echo "On MacOS, drag and drop the mitmproxy-ca.pem file into the Keychain Access app while having the 'Login' keychain selected."
+    echo "Then, double-click the certificate in the keychain and set ‘Always Trust‘ in the ‘Trust‘ section."
+    echo "The certificate is located at: ${mitmproxy_ca_pem_file}"
+    echo "After you're done, press enter to continue."
+    read -r
+fi
 
 echo "Generating the certificate bundle mitmproxy-cert-bundle.pem to pass to Python..."
 cp "$(python -m certifi)" mitmproxy-cert-bundle.pem
diff --git a/tests_integration/test_integration.py b/tests_integration/test_integration.py
