diff --git a/.env.docker-compose.dev b/.env.docker-compose.dev index ba4f907456..017fef7222 100644 --- a/.env.docker-compose.dev +++ b/.env.docker-compose.dev @@ -1,18 +1,14 @@ # garage admin token for init script GARAGE_ADMIN_TOKEN=dev_admin_token -ASSETS_BUCKET_NAME=assets.v7.pubpub.org -ASSETS_UPLOAD_KEY=pubpubuser -ASSETS_UPLOAD_SECRET_KEY=pubpubpass -# set to same as above for s3fs/caddy to work -AWS_ACCESS_KEY_ID=pubpubuser -AWS_SECRET_ACCESS_KEY=pubpubpass - -ASSETS_REGION=garage +S3_BUCKET_NAME=assets.v7.pubpub.org +S3_ACCESS_KEY=pubpubuser +S3_SECRET_KEY=pubpubpass +S3_REGION=garage # internal endpoint used by backend services running in Docker -ASSETS_STORAGE_ENDPOINT=http://garage:3900 +S3_ENDPOINT=http://minio:3900 # public endpoint used for signed URLs accessible from browsers -ASSETS_PUBLIC_ENDPOINT=http://localhost:3900 +S3_PUBLIC_ENDPOINT=http://localhost:3900 POSTGRES_PORT=54322 POSTGRES_USER=postgres diff --git a/.env.example b/.env.example new file mode 100644 index 0000000000..db873a652b --- /dev/null +++ b/.env.example @@ -0,0 +1,40 @@ +# Base environment configuration +# Copy this to .env and customize as needed +# Values here are defaults that work across development, testing, and self-hosting + +# Database configuration +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_DB=postgres +POSTGRES_PORT=54322 + +# Cache configuration +VALKEY_HOST=localhost +VALKEY_PORT=6379 + +# Minio configuration +MINIO_ROOT_USER=pubpub-admin +MINIO_ROOT_PASSWORD=pubpub-admin +S3_BUCKET_NAME=assets.pubpub.local +S3_ACCESS_KEY=pubpubuser +S3_SECRET_KEY=pubpubpass +S3_REGION=us-east-1 +S3_ENDPOINT=http://localhost:9000 + +# Email configuration +MAILGUN_SMTP_HOST=localhost +MAILGUN_SMTP_PORT=54325 +MAILGUN_SMTP_USERNAME=xxx +MAILGUN_SMTP_PASSWORD=xxx + +# Application configuration +API_KEY=super_secret_key +PUBPUB_URL=http://localhost:3000 + +# Other configuration +OTEL_SERVICE_NAME=pubpub-v7-dev +HONEYCOMB_API_KEY=xxx + +# Volume types (can be overridden per environment) +DB_VOLUME_TYPE=postgres_data +MINIO_VOLUME_TYPE=minio_data \ No newline at end of file diff --git a/.github/workflows/awsdeploy.yml b/.github/workflows/awsdeploy.yml deleted file mode 100644 index 52459d83f9..0000000000 --- a/.github/workflows/awsdeploy.yml +++ /dev/null @@ -1,80 +0,0 @@ -# Based on https://docs.github.com/en/actions/deployment/deploying-to-your-cloud-provider/deploying-to-amazon-elastic-container-service - -name: aws ecs deploy - -on: - workflow_call: - inputs: - proper-name: - required: true - type: string - environment: - required: true - type: string - image-tag-override: # example: latest, 7037e37a18a379d583164441baff9e594cc479f8 - type: string # use this to force a container version. - secrets: - AWS_ACCESS_KEY_ID: - required: true - AWS_SECRET_ACCESS_KEY: - required: true - - #dispatch event means you can call it from the Github UI and set inputs on a form. - # MUST match the inputs of the workflow_call. - workflow_dispatch: - inputs: - proper-name: - required: true - type: string - environment: - required: true - type: string - image-tag-override: # example: latest, 7037e37a18a379d583164441baff9e594cc479f8 - type: string # use this to force a container version. - -jobs: - deploy-core: - uses: ./.github/workflows/deploy-template.yml - with: - service: core - environment: ${{ inputs.environment }} - proper-name: ${{ inputs.proper-name }} - image-tag-override: ${{ inputs.image-tag-override }} - secrets: - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - - deploy-jobs: - uses: ./.github/workflows/deploy-template.yml - needs: deploy-core - with: - service: jobs - environment: ${{ inputs.environment }} - proper-name: ${{ inputs.proper-name }} - image-tag-override: ${{ inputs.image-tag-override }} - secrets: - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - - deploy-bastion: - uses: ./.github/workflows/deploy-template.yml - with: - service: bastion - environment: ${{ inputs.environment }} - proper-name: ${{ inputs.proper-name }} - repo-name-override: pubpub-v7 - image-tag-override: ${{ inputs.image-tag-override }} - secrets: - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - - deploy-site-builder: - uses: ./.github/workflows/deploy-template.yml - with: - service: site-builder - environment: ${{ inputs.environment }} - proper-name: ${{ inputs.proper-name }} - image-tag-override: ${{ inputs.image-tag-override }} - secrets: - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/.github/workflows/deploy-template.yml b/.github/workflows/deploy-template.yml deleted file mode 100644 index d98a0370c8..0000000000 --- a/.github/workflows/deploy-template.yml +++ /dev/null @@ -1,137 +0,0 @@ -# Based on https://docs.github.com/en/actions/deployment/deploying-to-your-cloud-provider/deploying-to-amazon-elastic-container-service - -name: aws ecs deploy template - -on: - workflow_call: - inputs: - service: # example: core - required: true - type: string - proper-name: # example: blake - required: true - type: string - environment: # example: staging - required: true - type: string - repo-name-override: - type: string - image-tag-override: # example: latest, 7037e37a18a379d583164441baff9e594cc479f8 - type: string # use this to force a container version. - secrets: - AWS_ACCESS_KEY_ID: - required: true - AWS_SECRET_ACCESS_KEY: - required: true - workflow_dispatch: - inputs: - service: # example: core - required: true - type: string - proper-name: # example: blake - required: true - type: string - environment: # example: staging - required: true - type: string - repo-name-override: - type: string - image-tag-override: # example: latest, 7037e37a18a379d583164441baff9e594cc479f8 - type: string # use this to force a container version. - -env: - AWS_REGION: us-east-1 - ECR_REPOSITORY_PREFIX: pubpub-v7 - ECR_REPOSITORY_NAME_OVERRIDE: ${{ inputs.repo-name-override }} - ECS_SERVICE: ${{ inputs.proper-name }}-${{inputs.service}} - ECS_CLUSTER: ${{inputs.proper-name}}-ecs-cluster-${{inputs.environment}} - ECS_TASK_DEFINITION_TEMPLATE: ${{ inputs.proper-name }}-${{inputs.service}} - CONTAINER_NAME: ${{inputs.service}} - -jobs: - deploy: - name: Deploy - runs-on: ubuntu-latest - environment: ${{ inputs.proper-name }}-${{ inputs.environment }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ vars.IAM_ROLE_TO_ASSUME }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.AWS_REGION }} - - - name: Get image tag based on SHA - id: gettag - env: - OVERRIDE: ${{inputs.image-tag-override}} - # use shell substitution - run: echo "tag=${OVERRIDE:-$(git describe --always --abbrev=40 --dirty)}" >> $GITHUB_OUTPUT - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Retrieve Task Definition contents from template - id: get-taskdef - run: | - aws ecs describe-task-definition \ - --task-definition $ECS_TASK_DEFINITION_TEMPLATE \ - --query taskDefinition >> template_task_def.json - - - name: Get image labels - id: label - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - IMAGE_TAG: ${{ steps.gettag.outputs.tag }} - run: | - echo "label=$ECR_REGISTRY/${ECR_REPOSITORY_NAME_OVERRIDE:-$ECR_REPOSITORY_PREFIX-${CONTAINER_NAME}}:$IMAGE_TAG" >> $GITHUB_OUTPUT - echo "base_label=$ECR_REGISTRY/$ECR_REPOSITORY_PREFIX:$IMAGE_TAG" >> $GITHUB_OUTPUT - - - name: Fill in the new image ID in the Amazon ECS task definition - id: task-def-service - uses: aws-actions/amazon-ecs-render-task-definition@c804dfbdd57f713b6c079302a4c01db7017a36fc - with: - task-definition: template_task_def.json - container-name: ${{ env.CONTAINER_NAME }} - image: ${{ steps.label.outputs.label }} - - # Complication when the number of containers in the task are unknown: - # we have to know where to get the inputs for each step, including the upload - # step. - - name: Fill in the new image ID in the Amazon ECS task definition for migrations - id: task-def-migration - if: inputs.service == 'core' - uses: aws-actions/amazon-ecs-render-task-definition@c804dfbdd57f713b6c079302a4c01db7017a36fc - with: - task-definition: ${{ steps.task-def-service.outputs.task-definition }} - container-name: migrations - image: ${{ steps.label.outputs.base_label }} - - - name: Deploy Amazon ECS task definition - id: deploy-service-only - # This one is different. The single-image case is when not deploying core. - if: inputs.service != 'core' - uses: aws-actions/amazon-ecs-deploy-task-definition@16f052ed696e6e5bf88c208a8e5ba1af7ced3310 - with: - # it is because of this line that the two steps need different if conditions - task-definition: ${{ steps.task-def-service.outputs.task-definition }} - service: ${{ env.ECS_SERVICE }} - cluster: ${{ env.ECS_CLUSTER }} - wait-for-service-stability: true - - - name: Deploy Amazon ECS task definition including migrations - id: deploy-service-and-migrations - if: inputs.service == 'core' - uses: aws-actions/amazon-ecs-deploy-task-definition@16f052ed696e6e5bf88c208a8e5ba1af7ced3310 - with: - # it is because of this line that the two steps need different if conditions - task-definition: ${{ steps.task-def-migration.outputs.task-definition }} - service: ${{ env.ECS_SERVICE }} - cluster: ${{ env.ECS_CLUSTER }} - wait-for-service-stability: true diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000000..8e9d34f39c --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,250 @@ +name: Deploy to Hetzner + +run-name: >- + ${{ + github.event_name == 'release' && format('Deploy prod: {0}', github.event.release.tag_name) || + github.event_name == 'workflow_run' && format('Deploy staging: {0}', github.event.workflow_run.head_commit.message) || + format('Deploy: {0}', github.sha) + }} + +concurrency: + group: >- + deploy-${{ + github.event_name == 'release' && format('release-{0}', github.event.release.tag_name) || + github.event_name == 'workflow_run' && format('ci-{0}', github.event.workflow_run.head_branch) || + github.ref + }} + cancel-in-progress: true + +on: + workflow_run: + workflows: [CI] + types: [completed] + branches: [main] + workflow_dispatch: + inputs: + skip_ci_check: + description: Deploy even if CI failed + required: false + default: false + type: boolean + release: + types: [published] + +jobs: + build: + runs-on: ubuntu-latest + if: | + github.event_name == 'workflow_dispatch' || + github.event_name == 'release' || + (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') + + permissions: + contents: read + packages: write + + outputs: + image_tag: ${{ steps.vars.outputs.image_tag }} + host: ${{ steps.vars.outputs.host }} + env_file: ${{ steps.vars.outputs.env_file }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.workflow_run.head_sha || github.sha }} + + - name: Set deployment vars + id: vars + run: | + if [[ "${{ github.event_name }}" == "release" ]]; then + echo "image_tag=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT + echo "host=${{ secrets.SSH_HOST_PROD }}" >> $GITHUB_OUTPUT + echo "env_file=.env.enc" >> $GITHUB_OUTPUT + echo "publish_latest=false" >> $GITHUB_OUTPUT + else + echo "image_tag=${{ github.sha }}" >> $GITHUB_OUTPUT + echo "host=${{ secrets.SSH_HOST_STAGING }}" >> $GITHUB_OUTPUT + echo "env_file=.env.staging.enc" >> $GITHUB_OUTPUT + echo "publish_latest=true" >> $GITHUB_OUTPUT + fi + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push pubstar + uses: docker/build-push-action@v6 + with: + context: . + push: true + provenance: false + sbom: false + cache-from: type=gha,scope=platform + cache-to: type=gha,mode=max,scope=platform + build-args: | + PACKAGE=core + CI=true + secrets: | + SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} + target: next-app-core + tags: | + ghcr.io/knowledgefutures/platform:${{ steps.vars.outputs.image_tag }} + ${{ steps.vars.outputs.publish_latest == 'true' && 'ghcr.io/knowledgefutures/platform:latest' || '' }} + platforms: linux/amd64 + + - name: Build and push platform-jobs + uses: docker/build-push-action@v6 + with: + context: . + push: true + provenance: false + sbom: false + cache-from: type=gha,scope=platform-jobs + cache-to: type=gha,mode=max,scope=platform-jobs + build-args: | + PACKAGE=jobs + CI=true + secrets: | + SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} + target: jobs + tags: | + ghcr.io/knowledgefutures/platform-jobs:${{ steps.vars.outputs.image_tag }} + ${{ steps.vars.outputs.publish_latest == 'true' && 'ghcr.io/knowledgefutures/platform-jobs:latest' || '' }} + platforms: linux/amd64 + + - name: Build and push platform-site-builder + uses: docker/build-push-action@v6 + with: + context: . + push: true + provenance: false + sbom: false + cache-from: type=gha,scope=platform-site-builder + cache-to: type=gha,mode=max,scope=platform-site-builder + build-args: | + PACKAGE=site-builder + CI=true + secrets: | + SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} + target: jobs + tags: | + ghcr.io/knowledgefutures/platform-site-builder:${{ steps.vars.outputs.image_tag }} + ${{ steps.vars.outputs.publish_latest == 'true' && 'ghcr.io/knowledgefutures/platform-site-builder:latest' || '' }} + platforms: linux/amd64 + + deploy: + needs: build + runs-on: ubuntu-latest + + steps: + - name: Start SSH agent + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: Add known hosts + run: | + mkdir -p ~/.ssh + ssh-keyscan -H "${{ needs.build.outputs.host }}" >> ~/.ssh/known_hosts + + - name: Deploy over SSH + env: + SSH_USER: ${{ secrets.SSH_USER }} + SSH_HOST: ${{ needs.build.outputs.host }} + REPO: ${{ github.repository }} + BRANCH: ${{ github.ref_name }} + GHCR_USER: ${{ secrets.GHCR_USER }} + GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} + IMAGE_TAG: ${{ needs.build.outputs.image_tag }} + ENV_FILE: ${{ needs.build.outputs.env_file }} + run: | + ssh "${SSH_USER}@${SSH_HOST}" \ + "env GHCR_USER='${GHCR_USER}' GHCR_TOKEN='${GHCR_TOKEN}' IMAGE_TAG='${IMAGE_TAG}' ENV_FILE='${ENV_FILE}' bash -s -- '${REPO}' '${BRANCH}'" <<'EOS' + set -euo pipefail + + REPO="${1:?missing repo}" + BRANCH="${2:-main}" + + : "${IMAGE_TAG:?missing IMAGE_TAG}" + : "${GHCR_USER:?missing GHCR_USER}" + : "${GHCR_TOKEN:?missing GHCR_TOKEN}" + + REPO_NAME="${REPO##*/}" + APP_DIR="/srv/${REPO_NAME}" + REPO_SSH="git@github.com:${REPO}.git" + + if [[ -z "$REPO_NAME" || -z "$APP_DIR" ]]; then + echo "bad derived paths: REPO='$REPO' REPO_NAME='$REPO_NAME' APP_DIR='$APP_DIR'" + exit 1 + fi + + ssh-keyscan -H github.com >> ~/.ssh/known_hosts 2>/dev/null + chmod 600 ~/.ssh/known_hosts + + if [[ ! -d "${APP_DIR}/.git" ]]; then + sudo mkdir -p "${APP_DIR}" + sudo chown -R "$USER:$USER" "${APP_DIR}" + git clone --branch "${BRANCH}" "${REPO_SSH}" "${APP_DIR}" + fi + + cd "${APP_DIR}" + git fetch --prune --tags origin + git checkout --detach "${IMAGE_TAG}" 2>/dev/null || git checkout --detach "origin/${BRANCH}" + + cd infra + umask 077 + + : "${ENV_FILE:?missing ENV_FILE}" + sops -d --input-type dotenv --output-type dotenv "$ENV_FILE" > .env + + if ! sudo docker info --format '{{.Swarm.LocalNodeState}}' | grep -qx active; then + sudo docker swarm init --advertise-addr "$(hostname -I | awk '{print $1}')" + fi + + echo "$GHCR_TOKEN" | sudo docker login ghcr.io -u "$GHCR_USER" --password-stdin + + echo "deploying with IMAGE_TAG=$IMAGE_TAG" + + sudo env IMAGE_TAG="$IMAGE_TAG" \ + docker stack deploy -c stack.yml \ + --with-registry-auth --resolve-image always --prune \ + pubstar + + sudo docker stack services pubstar + sudo docker image prune -f + + # wait for platform rollout + wait_rollout() { + echo "waiting for rollout of $1..." + svc="$1" + timeout="${2:-600}" + end=$((SECONDS+timeout)) + + while (( SECONDS < end )); do + desired="$(sudo docker service inspect "$svc" --format '{{.Spec.Mode.Replicated.Replicas}}' 2>/dev/null || echo "")" + running="$(sudo docker service ps "$svc" --filter desired-state=running --format '{{.CurrentState}}' 2>/dev/null | grep -c '^Running' || true)" + state="$(sudo docker service inspect "$svc" --format '{{if .UpdateStatus}}{{.UpdateStatus.State}}{{end}}' 2>/dev/null || echo "")" + echo " $svc: desired=$desired running=$running state=$state" + + if [[ -n "$desired" && "$running" == "$desired" ]] && { [[ -z "$state" ]] || [[ "$state" == "completed" ]]; }; then + echo " $svc rollout complete" + return 0 + fi + + sleep 5 + done + + echo "rollout timeout for $svc" + return 1 + } + + wait_rollout pubstar 600 + + EOS diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a49b764d75..d147f1eae0 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,26 +1,21 @@ on: workflow_call: inputs: - image-tag-override: # example: latest, 7037e37a18a379d583164441baff9e594cc479f8 - type: string # use this to force a container version. - secrets: - AWS_ACCESS_KEY_ID: - required: true - AWS_SECRET_ACCESS_KEY: - required: true + image-tag-override: + type: string env: CI: true - AWS_REGION: us-east-1 - - ECR_REPOSITORY_PREFIX: pubpub-v7 - CONTAINER_NAME: core jobs: integration-tests: name: Integration tests runs-on: ubuntu-latest + permissions: + contents: read + packages: read + strategy: matrix: package: @@ -52,7 +47,6 @@ jobs: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_OUTPUT - name: Setup pnpm cache - # since this always runs after CI, there's no need to save the cache afterwards, since it's guaranteed to be the same uses: actions/cache/restore@v4 with: path: ${{ steps.get-store-path.outputs.STORE_PATH }} @@ -60,7 +54,6 @@ jobs: restore-keys: | ${{ runner.os }}-pnpm-store- - # mostly to skip preconstruct build - name: Setup turbo cache uses: actions/cache/restore@v4 with: @@ -69,15 +62,7 @@ jobs: restore-keys: | ${{ runner.os }}-turbo- - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ vars.IAM_ROLE_TO_ASSUME }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.AWS_REGION }} - - - name: Get image tag based on SHA + - name: Get image tag id: gettag run: | if [ -n "${{ inputs.image-tag-override }}" ]; then @@ -88,20 +73,21 @@ jobs: echo "Using current SHA as image tag" fi - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Get image labels + - name: Compute image refs id: label env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} IMAGE_TAG: ${{ steps.gettag.outputs.tag }} run: | - echo "core_label=$ECR_REGISTRY/${ECR_REPOSITORY_NAME_OVERRIDE:-$ECR_REPOSITORY_PREFIX-core}:$IMAGE_TAG" >> $GITHUB_OUTPUT - echo "jobs_label=$ECR_REGISTRY/${ECR_REPOSITORY_NAME_OVERRIDE:-$ECR_REPOSITORY_PREFIX-jobs}:$IMAGE_TAG" >> $GITHUB_OUTPUT - echo "base_label=$ECR_REGISTRY/$ECR_REPOSITORY_PREFIX:$IMAGE_TAG" >> $GITHUB_OUTPUT - echo "site_builder_label=$ECR_REGISTRY/${ECR_REPOSITORY_NAME_OVERRIDE:-$ECR_REPOSITORY_PREFIX-site-builder}:$IMAGE_TAG" >> $GITHUB_OUTPUT + echo "core_label=ghcr.io/knowledgefutures/platform:$IMAGE_TAG" >> $GITHUB_OUTPUT + echo "jobs_label=ghcr.io/knowledgefutures/platform-jobs:$IMAGE_TAG" >> $GITHUB_OUTPUT + echo "site_builder_label=ghcr.io/knowledgefutures/platform-site-builder:$IMAGE_TAG" >> $GITHUB_OUTPUT - name: Install dependencies run: pnpm install --frozen-lockfile --prefer-offline @@ -115,7 +101,6 @@ jobs: - name: Run migrations and seed run: pnpm --filter core reset-base env: - # 20241126: this prevents the arcadia seed from running, which contains a ton of pubs which potentially might slow down the tests MINIMAL_SEED: true SKIP_VALIDATION: true @@ -124,16 +109,16 @@ jobs: - name: Start up core etc run: pnpm integration:setup env: - INTEGRATION_TESTS_IMAGE: ${{steps.label.outputs.core_label}} - SITE_BUILDER_IMAGE: ${{steps.label.outputs.site_builder_label}} - JOBS_IMAGE: ${{steps.label.outputs.jobs_label}} + INTEGRATION_TESTS_IMAGE: ${{ steps.label.outputs.core_label }} + SITE_BUILDER_IMAGE: ${{ steps.label.outputs.site_builder_label }} + JOBS_IMAGE: ${{ steps.label.outputs.jobs_label }} - name: Log out Container ID for health check id: log-container-id run: echo "CONTAINER_ID=$(docker compose -f docker-compose.test.yml ps integration-tests -q)" >> $GITHUB_OUTPUT - name: Wait until container is healthy - run: while [ "`docker inspect -f {{.State.Health.Status}} ${{steps.log-container-id.outputs.CONTAINER_ID}}`" != "healthy" ]; do sleep .2; done + run: while [ "`docker inspect -f {{.State.Health.Status}} ${{ steps.log-container-id.outputs.CONTAINER_ID }}`" != "healthy" ]; do sleep .2; done - name: Run integration tests run: pnpm playwright:test --filter ${{ matrix.package }} --env-mode=loose @@ -143,12 +128,12 @@ jobs: DATABASE_URL: postgresql://postgres:postgres@localhost:54322/postgres - name: Print container logs - if: ${{failure() || cancelled()}} + if: ${{ failure() || cancelled() }} run: docker compose -f docker-compose.test.yml --profile integration logs -t env: - INTEGRATION_TESTS_IMAGE: ${{steps.label.outputs.core_label}} - SITE_BUILDER_IMAGE: ${{steps.label.outputs.site_builder_label}} - JOBS_IMAGE: ${{steps.label.outputs.jobs_label}} + INTEGRATION_TESTS_IMAGE: ${{ steps.label.outputs.core_label }} + SITE_BUILDER_IMAGE: ${{ steps.label.outputs.site_builder_label }} + JOBS_IMAGE: ${{ steps.label.outputs.jobs_label }} - name: Upload core playwright snapshots artifact if: failure() && matrix.package == 'core' diff --git a/.github/workflows/ecrbuild-all.yml b/.github/workflows/ecrbuild-all.yml deleted file mode 100644 index 11d9af56da..0000000000 --- a/.github/workflows/ecrbuild-all.yml +++ /dev/null @@ -1,89 +0,0 @@ -# Based on https://docs.github.com/en/actions/deployment/deploying-to-your-cloud-provider/deploying-to-amazon-elastic-container-service - -name: docker build to ECR - -on: - workflow_call: - secrets: - AWS_ACCESS_KEY_ID: - required: true - AWS_SECRET_ACCESS_KEY: - required: true - inputs: - publish_to_ghcr: - type: boolean - default: false - outputs: - core-image: - description: "Core image SHA" - value: ${{ jobs.build-core.outputs.image-sha }} - base-image: - description: "Base image SHA" - value: ${{ jobs.build-base.outputs.image-sha }} - jobs-image: - description: "Jobs image SHA" - value: ${{ jobs.build-jobs.outputs.image-sha }} - site-builder-image: - description: "Site builder image SHA" - value: ${{ jobs.build-site-builder.outputs.image-sha }} - -jobs: - emit-sha-tag: - name: Emit container tag sha - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha || github.sha }} - - name: Get image tag - id: label - run: | - sha_short=$(git describe --always --abbrev=40 --dirty) - echo "Building containers with tag:" - echo "$sha_short" - - build-base: - uses: ./.github/workflows/ecrbuild-template.yml - with: - publish_to_ghcr: ${{ inputs.publish_to_ghcr }} - ghcr_image_name: platform-migrations - secrets: - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - - build-core: - uses: ./.github/workflows/ecrbuild-template.yml - # needs: - # - build-base - with: - package: core - publish_to_ghcr: ${{ inputs.publish_to_ghcr }} - ghcr_image_name: platform - secrets: - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - - build-jobs: - uses: ./.github/workflows/ecrbuild-template.yml - # needs: - # - build-base - with: - package: jobs - target: jobs - publish_to_ghcr: ${{ inputs.publish_to_ghcr }} - ghcr_image_name: platform-jobs - secrets: - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - - build-site-builder: - uses: ./.github/workflows/ecrbuild-template.yml - with: - package: site-builder - target: jobs - publish_to_ghcr: ${{ inputs.publish_to_ghcr }} - ghcr_image_name: platform-site-builder - secrets: - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} diff --git a/.github/workflows/ecrbuild-template.yml b/.github/workflows/ecrbuild-template.yml deleted file mode 100644 index 4fc87f6027..0000000000 --- a/.github/workflows/ecrbuild-template.yml +++ /dev/null @@ -1,143 +0,0 @@ -# Based on https://docs.github.com/en/actions/deployment/deploying-to-your-cloud-provider/deploying-to-amazon-elastic-container-service - -name: aws ecr build template - -on: - workflow_call: - inputs: - package: - type: string - runner: - type: string - default: ubuntu-latest - target: - type: string - publish_to_ghcr: - type: boolean - default: false - ghcr_image_name: - type: string - required: false - outputs: - image-sha: - description: "Image SHA" - value: ${{ jobs.build.outputs.image-sha }} - secrets: - AWS_ACCESS_KEY_ID: - required: true - AWS_SECRET_ACCESS_KEY: - required: true - -env: - PACKAGE: ${{ inputs.package }} - AWS_REGION: us-east-1 # set this to your preferred AWS region, e.g. us-west-1 - ECR_REPOSITORY_PREFIX: pubpub-v7 # set this to your Amazon ECR repository name - TARGET: ${{ inputs.target }} - -jobs: - build: - name: Build - runs-on: ${{ inputs.runner }} - outputs: - image-sha: ${{ steps.label.outputs.label }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha || github.sha }} - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ vars.IAM_ROLE_TO_ASSUME }} - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.AWS_REGION }} - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # necessary in order to upload build source maps to sentry - - name: Get sentry token - id: sentry-token - uses: aws-actions/aws-secretsmanager-get-secrets@v2 - with: - secret-ids: | - SENTRY_AUTH_TOKEN, ${{ vars.SENTRY_AUTH_TOKEN_ARN }} - - - name: setup docker buildx - uses: docker/setup-buildx-action@v3 - - - name: Create and use a new builder instance - run: | - docker buildx create --name cached-builder --use - - - name: Get image label - id: label - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - run: | - sha_short=$(git describe --always --abbrev=40 --dirty) - if [[ -z $PACKAGE ]] - then - package_suffix="" - echo "target=monorepo" >> $GITHUB_OUTPUT - else - package_suffix="-${PACKAGE}" - echo "target=${TARGET:-next-app-${PACKAGE}}" >> $GITHUB_OUTPUT - fi - echo "label=$ECR_REGISTRY/$ECR_REPOSITORY_PREFIX$package_suffix:$sha_short" >> $GITHUB_OUTPUT - if [[ ${{ inputs.publish_to_ghcr }} == "true" && -n ${{ inputs.ghcr_image_name }} ]] - then - TIMESTAMP=$(date +%Y%m%d-%H%M%S) - - echo "ghcr_latest_label=ghcr.io/pubpub/${{ inputs.ghcr_image_name }}:latest" >> $GITHUB_OUTPUT - - echo "ghcr_sha_label=ghcr.io/pubpub/${{ inputs.ghcr_image_name }}:$sha_short" >> $GITHUB_OUTPUT - - echo "ghcr_timestamp_label=ghcr.io/pubpub/${{ inputs.ghcr_image_name }}:$TIMESTAMP" >> $GITHUB_OUTPUT - fi - - - name: Check if SENTRY_AUTH_TOKEN is set - run: | - if [[ -z ${{ env.SENTRY_AUTH_TOKEN }} ]] - then - echo "SENTRY_AUTH_TOKEN is not set" - exit 1 - fi - - - name: Build, tag, and push image to Amazon ECR - uses: docker/build-push-action@v6 - id: build-image - env: - REGISTRY_REF: ${{steps.login-ecr.outputs.registry}}/${{env.ECR_REPOSITORY_PREFIX}}-${{env.PACKAGE}}:cache - LABEL: ${{ steps.label.outputs.label }} - TARGET: ${{ steps.label.outputs.target }} - SENTRY_AUTH_TOKEN: ${{ env.SENTRY_AUTH_TOKEN }} - with: - context: . - # cache-from: type=registry,ref=${{env.REGISTRY_REF}} - # cache-to: type=registry,mode=max,image-manifest=true,oci-mediatypes=true,ref=${{env.REGISTRY_REF}} - builder: cached-builder - build-args: | - PACKAGE=${{ inputs.package }} - CI=true - secrets: | - SENTRY_AUTH_TOKEN=${{ env.SENTRY_AUTH_TOKEN }} - target: ${{ steps.label.outputs.target }} - tags: | - ${{ steps.label.outputs.label }} - ${{ steps.label.outputs.ghcr_latest_label }} - ${{ steps.label.outputs.ghcr_sha_label }} - ${{ steps.label.outputs.ghcr_timestamp_label }} - platforms: linux/amd64 - push: true diff --git a/.github/workflows/ghcr-build-all.yml b/.github/workflows/ghcr-build-all.yml new file mode 100644 index 0000000000..00081a272e --- /dev/null +++ b/.github/workflows/ghcr-build-all.yml @@ -0,0 +1,51 @@ +name: docker build to GHCR + +on: + workflow_call: + inputs: + publish_latest: + type: boolean + default: false + outputs: + core-image: + description: 'Core image ref' + value: ${{ jobs.build-core.outputs.image-sha }} + jobs-image: + description: 'Jobs image ref' + value: ${{ jobs.build-jobs.outputs.image-sha }} + site-builder-image: + description: 'Site builder image ref' + value: ${{ jobs.build-site-builder.outputs.image-sha }} + secrets: + SENTRY_AUTH_TOKEN: + required: true + +jobs: + build-core: + uses: ./.github/workflows/ghcr-build-template.yml + with: + package: core + ghcr_image_name: platform + publish_latest: ${{ inputs.publish_latest }} + secrets: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + + build-jobs: + uses: ./.github/workflows/ghcr-build-template.yml + with: + package: jobs + target: jobs + ghcr_image_name: platform-jobs + publish_latest: ${{ inputs.publish_latest }} + secrets: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + + build-site-builder: + uses: ./.github/workflows/ghcr-build-template.yml + with: + package: site-builder + target: jobs + ghcr_image_name: platform-site-builder + publish_latest: ${{ inputs.publish_latest }} + secrets: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} diff --git a/.github/workflows/ghcr-build-template.yml b/.github/workflows/ghcr-build-template.yml new file mode 100644 index 0000000000..32da2fa6f3 --- /dev/null +++ b/.github/workflows/ghcr-build-template.yml @@ -0,0 +1,89 @@ +name: ghcr build template + +on: + workflow_call: + inputs: + package: + type: string + runner: + type: string + default: ubuntu-latest + target: + type: string + ghcr_image_name: + type: string + required: true + publish_latest: + type: boolean + default: false + outputs: + image-sha: + description: 'Full GHCR image ref with SHA tag' + value: ${{ jobs.build.outputs.image-sha }} + secrets: + SENTRY_AUTH_TOKEN: + required: true + +jobs: + build: + name: Build + runs-on: ${{ inputs.runner }} + permissions: + contents: read + packages: write + + outputs: + image-sha: ${{ steps.label.outputs.label }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Compute image tags + id: label + run: | + sha_short=$(git describe --always --abbrev=40 --dirty) + + if [[ -z "${{ inputs.package }}" ]]; then + echo "target=monorepo" >> $GITHUB_OUTPUT + else + echo "target=${{ inputs.target || format('next-app-{0}', inputs.package) }}" >> $GITHUB_OUTPUT + fi + + echo "label=ghcr.io/knowledgefutures/${{ inputs.ghcr_image_name }}:$sha_short" >> $GITHUB_OUTPUT + + TAGS="ghcr.io/knowledgefutures/${{ inputs.ghcr_image_name }}:$sha_short" + if [[ "${{ inputs.publish_latest }}" == "true" ]]; then + TAGS="$TAGS,ghcr.io/knowledgefutures/${{ inputs.ghcr_image_name }}:latest" + fi + echo "tags=$TAGS" >> $GITHUB_OUTPUT + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + cache-from: type=gha,scope=${{ inputs.ghcr_image_name }} + cache-to: type=gha,mode=max,scope=${{ inputs.ghcr_image_name }} + build-args: | + PACKAGE=${{ inputs.package }} + CI=true + secrets: | + SENTRY_AUTH_TOKEN=${{ secrets.SENTRY_AUTH_TOKEN }} + target: ${{ steps.label.outputs.target }} + tags: ${{ steps.label.outputs.tags }} + platforms: linux/amd64 + push: true + provenance: false + sbom: false diff --git a/.github/workflows/on_main.yml b/.github/workflows/on_main.yml index 265f1ae985..3b287557ec 100644 --- a/.github/workflows/on_main.yml +++ b/.github/workflows/on_main.yml @@ -1,5 +1,3 @@ -# Based on https://docs.github.com/en/actions/deployment/deploying-to-your-cloud-provider/deploying-to-amazon-elastic-container-service - name: Promote from main on: @@ -7,39 +5,30 @@ on: branches: - main +permissions: + contents: read + packages: write + jobs: ci: uses: ./.github/workflows/ci.yml build-all: needs: ci - uses: ./.github/workflows/ecrbuild-all.yml - with: - publish_to_ghcr: true + permissions: + contents: read + packages: write + uses: ./.github/workflows/ghcr-build-all.yml secrets: - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} + with: + publish_latest: true run-e2e: needs: - ci - build-all uses: ./.github/workflows/e2e.yml - secrets: - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - - deploy-all: - uses: ./.github/workflows/awsdeploy.yml - needs: - - build-all - - run-e2e - with: - proper-name: stevie - environment: production - secrets: - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} deploy-docs: permissions: @@ -49,27 +38,3 @@ jobs: uses: ./.github/workflows/build-docs.yml with: preview: false - - deploy-preview: - uses: ./.github/workflows/pull-preview.yml - needs: - - build-all - permissions: - contents: read - deployments: write - pull-requests: write - statuses: write - with: - PLATFORM_IMAGE: ${{ needs.build-all.outputs.core-image }} - JOBS_IMAGE: ${{ needs.build-all.outputs.jobs-image }} - MIGRATIONS_IMAGE: ${{ needs.build-all.outputs.base-image }} - SITE_BUILDER_IMAGE: ${{ needs.build-all.outputs.site-builder-image }} - AWS_REGION: "us-east-1" - ALWAYS_ON: "main" - COMPOSE_FILES: docker-compose.preview.sandbox.yml - secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GH_PAT_PR_PREVIEW_CLEANUP: ${{ secrets.GH_PAT_PR_PREVIEW_CLEANUP }} - PREVIEW_DATACITE_REPOSITORY_ID: ${{ secrets.PREVIEW_DATACITE_REPOSITORY_ID }} - PREVIEW_DATACITE_PASSWORD: ${{ secrets.PREVIEW_DATACITE_PASSWORD }} diff --git a/.github/workflows/on_pr.yml b/.github/workflows/on_pr.yml index 12a513f65e..ecdbff70ff 100644 --- a/.github/workflows/on_pr.yml +++ b/.github/workflows/on_pr.yml @@ -1,22 +1,18 @@ -# Based on https://docs.github.com/en/actions/deployment/deploying-to-your-cloud-provider/deploying-to-amazon-elastic-container-service - name: PR Updated triggers on: pull_request: types: [labeled, unlabeled, synchronize, closed, reopened, opened] -env: - AWS_REGION: us-east-1 - permissions: id-token: write contents: read + packages: write jobs: path-filter: runs-on: ubuntu-latest - if: github.event.action == 'opened' || github.event.action == 'reopened' || github.event.action == 'synchronize' || github.event.action == 'closed' + if: github.event.action == 'opened' || github.event.action == 'reopened' || github.event.action == 'synchronize' || github.event.action == 'closed' || github.event.action == 'labeled' || github.event.action == 'unlabeled' outputs: docs: ${{ steps.changes.outputs.docs }} steps: @@ -30,7 +26,6 @@ jobs: docs: - 'docs/**' - # you can skip the build by adding 'skip-build' to the commit message, useful when testing tests skip_build_sha: outputs: @@ -62,20 +57,17 @@ jobs: run: | pr_number="${{ github.event.pull_request.number }}" - # get all workflow runs for this PR gh api "/repos/${{ github.repository }}/actions/workflows/on_pr.yml/runs?event=pull_request&per_page=100" \ --jq ".workflow_runs[] | select(.pull_requests[]?.number == ${pr_number}) | select(.id < ${{ github.run_id }}) | {id: .id, sha: .head_sha, created: .created_at}" \ | jq -s 'sort_by(.created) | reverse | .[].id' -r \ | while read run_id; do echo "Checking run: $run_id" - # check if build-all job succeeded in this run run=$(gh api "/repos/${{ github.repository }}/actions/runs/${run_id}/jobs") echo "Run: $run" all_success=$(echo "$run" | jq '[.jobs[] | select(.name | contains("build-all")) | .conclusion] | all(. == "success")') echo "All success for $run_id: $all_success" if [ "$all_success" == "true" ]; then - # get the SHA for this run successful_sha=$(gh api "/repos/${{ github.repository }}/actions/runs/${run_id}" --jq '.head_sha') echo "last-successful-build-sha=${successful_sha}" >> $GITHUB_OUTPUT echo "Found last successful build at SHA: $successful_sha (run: $run_id)" @@ -97,10 +89,12 @@ jobs: needs: - path-filter - skip_build_sha - uses: ./.github/workflows/ecrbuild-all.yml + permissions: + contents: read + packages: write + uses: ./.github/workflows/ghcr-build-all.yml secrets: - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} e2e: if: (github.event.action == 'opened' || github.event.action == 'reopened' || github.event.action == 'synchronize') && always() && (needs.build-all.result == 'success' || needs.build-all.result == 'skipped') @@ -111,88 +105,70 @@ jobs: uses: ./.github/workflows/e2e.yml with: image-tag-override: ${{ needs.skip_build_sha.outputs.last-successful-build-sha || '' }} - secrets: - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} deploy-preview: - if: needs.build-all.result == 'success' - uses: ./.github/workflows/pull-preview.yml + if: needs.build-all.result == 'success' && contains(github.event.pull_request.labels.*.name, 'preview') + uses: ./.github/workflows/preview.yml needs: - build-all permissions: contents: read - deployments: write pull-requests: write - statuses: write with: - # PLATFORM_IMAGE: 246372085946.dkr.ecr.us-east-1.amazonaws.com/pubpub-v7-core:2b9a81a279c4e405bbedcdbb697c897ded52fbc0 - # JOBS_IMAGE: 246372085946.dkr.ecr.us-east-1.amazonaws.com/pubpub-v7-jobs:c786662f4899de16a621e366a485eca5adda4d6a - # MIGRATIONS_IMAGE: 246372085946.dkr.ecr.us-east-1.amazonaws.com/pubpub-v7:c786662f4899de16a621e366a485eca5adda4d6a - # SITE_BUILDER_IMAGE: 246372085946.dkr.ecr.us-east-1.amazonaws.com/pubpub-v7-site-builder:c786662f4899de16a621e366a485eca5adda4d6a - PLATFORM_IMAGE: ${{ needs.build-all.outputs.core-image }} - JOBS_IMAGE: ${{ needs.build-all.outputs.jobs-image }} - MIGRATIONS_IMAGE: ${{ needs.build-all.outputs.base-image }} - SITE_BUILDER_IMAGE: ${{ needs.build-all.outputs.site-builder-image }} - AWS_REGION: "us-east-1" - COMPOSE_FILES: docker-compose.preview.pr.yml + action: deploy + image_tag: ${{ github.event.pull_request.head.sha }} + # image_tag: 1792116d0d1279eba2ad574741171449d50feb20 secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GH_PAT_PR_PREVIEW_CLEANUP: ${{ secrets.GH_PAT_PR_PREVIEW_CLEANUP }} - PREVIEW_DATACITE_REPOSITORY_ID: ${{ secrets.PREVIEW_DATACITE_REPOSITORY_ID }} - PREVIEW_DATACITE_PASSWORD: ${{ secrets.PREVIEW_DATACITE_PASSWORD }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + SSH_USER: ${{ secrets.SSH_USER }} + SSH_HOST_PREVIEW: ${{ secrets.SSH_HOST_PREVIEW }} + GHCR_USER: ${{ secrets.GHCR_USER }} + GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} close-preview: - uses: ./.github/workflows/pull-preview.yml - if: ${{(github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'preview')) || (github.event.action == 'unlabeled' && github.event.label.name == 'preview')}} + if: (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'preview')) || (github.event.action == 'unlabeled' && github.event.label.name == 'preview') + uses: ./.github/workflows/preview.yml permissions: contents: read - deployments: write pull-requests: write - statuses: write with: - PLATFORM_IMAGE: "x" # not used - JOBS_IMAGE: "x" # not used - MIGRATIONS_IMAGE: "x" # not used - SITE_BUILDER_IMAGE: "x" # not used - AWS_REGION: "us-east-1" + action: teardown secrets: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - GH_PAT_PR_PREVIEW_CLEANUP: ${{ secrets.GH_PAT_PR_PREVIEW_CLEANUP }} - PREVIEW_DATACITE_REPOSITORY_ID: ${{ secrets.PREVIEW_DATACITE_REPOSITORY_ID }} - PREVIEW_DATACITE_PASSWORD: ${{ secrets.PREVIEW_DATACITE_PASSWORD }} - - deploy-docs-preview: - permissions: - contents: write - pages: write - pull-requests: write - needs: - - path-filter - if: (github.event.action == 'opened' || github.event.action == 'reopened' || github.event.action == 'synchronize') && needs.path-filter.outputs.docs == 'true' - uses: ./.github/workflows/build-docs.yml - with: - preview: true - - close-docs-preview: - needs: - - path-filter - permissions: - contents: write - pages: write - pull-requests: write - if: github.event.action == 'closed' && needs.path-filter.outputs.docs == 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Close docs preview - uses: rossjrw/pr-preview-action@v1 - with: - source-dir: docs/out - action: remove + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + SSH_USER: ${{ secrets.SSH_USER }} + SSH_HOST_PREVIEW: ${{ secrets.SSH_HOST_PREVIEW }} + GHCR_USER: ${{ secrets.GHCR_USER }} + GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} + + # deploy-docs-preview: + # permissions: + # contents: write + # pages: write + # pull-requests: write + # needs: + # - path-filter + # if: (github.event.action == 'opened' || github.event.action == 'reopened' || github.event.action == 'synchronize') && needs.path-filter.outputs.docs == 'true' + # uses: ./.github/workflows/build-docs.yml + # with: + # preview: true + + # close-docs-preview: + # needs: + # - path-filter + # permissions: + # contents: write + # pages: write + # pull-requests: write + # if: github.event.action == 'closed' && needs.path-filter.outputs.docs == 'true' + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + + # - name: Close docs preview + # uses: rossjrw/pr-preview-action@v1 + # with: + # source-dir: docs/out + # action: remove status-check: needs: diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml new file mode 100644 index 0000000000..bec0c0ddb3 --- /dev/null +++ b/.github/workflows/preview.yml @@ -0,0 +1,209 @@ +name: PR Preview + +on: + workflow_call: + inputs: + action: + required: true + type: string + description: "'deploy' or 'teardown'" + image_tag: + required: false + type: string + description: "image tag to deploy (only needed for deploy)" + secrets: + SSH_PRIVATE_KEY: + required: true + SSH_USER: + required: true + SSH_HOST_PREVIEW: + required: true + GHCR_USER: + required: true + GHCR_TOKEN: + required: true + +permissions: + contents: read + pull-requests: write + +jobs: + preview: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + + - name: Get PR number + id: pr + run: | + echo "number=${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT + echo "stack_name=preview-pr-${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT + echo "host=pr-${{ github.event.pull_request.number }}.preview.pubstar.org" >> $GITHUB_OUTPUT + + - name: Start SSH agent + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: Add known hosts + run: | + mkdir -p ~/.ssh + ssh-keyscan -H "${{ secrets.SSH_HOST_PREVIEW }}" >> ~/.ssh/known_hosts + + - name: Deploy preview stack + if: inputs.action == 'deploy' + env: + SSH_USER: ${{ secrets.SSH_USER }} + SSH_HOST: ${{ secrets.SSH_HOST_PREVIEW }} + REPO: ${{ github.repository }} + BRANCH: ${{ github.head_ref }} + GHCR_USER: ${{ secrets.GHCR_USER }} + GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} + IMAGE_TAG: ${{ inputs.image_tag }} + STACK_NAME: ${{ steps.pr.outputs.stack_name }} + PREVIEW_HOST: ${{steps.pr.outputs.host }} + PR_NUMBER: ${{ steps.pr.outputs.number }} + run: | + ssh "${SSH_USER}@${SSH_HOST}" \ + "env GHCR_USER='${GHCR_USER}' GHCR_TOKEN='${GHCR_TOKEN}' IMAGE_TAG='${IMAGE_TAG}' STACK_NAME='${STACK_NAME}' PREVIEW_HOST='${PREVIEW_HOST}' PR_NUMBER='${PR_NUMBER}' bash -s -- '${REPO}' '${BRANCH}'" <<'EOS' + set -euo pipefail + + REPO="${1:?missing repo}" + BRANCH="${2:-main}" + + : "${IMAGE_TAG:?missing IMAGE_TAG}" + : "${GHCR_USER:?missing GHCR_USER}" + : "${GHCR_TOKEN:?missing GHCR_TOKEN}" + : "${STACK_NAME:?missing STACK_NAME}" + : "${PREVIEW_HOST:?missing PREVIEW_HOST}" + : "${PR_NUMBER:?missing PR_NUMBER}" + + REPO_NAME="${REPO##*/}" + APP_DIR="/srv/${REPO_NAME}" + REPO_SSH="git@github.com:${REPO}.git" + + ssh-keyscan -H github.com >> ~/.ssh/known_hosts 2>/dev/null + chmod 600 ~/.ssh/known_hosts + + if [[ ! -d "${APP_DIR}/.git" ]]; then + sudo mkdir -p "${APP_DIR}" + sudo chown -R "$USER:$USER" "${APP_DIR}" + git clone --branch "${BRANCH}" "${REPO_SSH}" "${APP_DIR}" + fi + + cd "${APP_DIR}" + git fetch --prune origin + git checkout "origin/${BRANCH}" --detach + + cd infra + + sops -d --input-type dotenv --output-type dotenv ".env.preview.enc" > .env + + if ! sudo docker info --format '{{.Swarm.LocalNodeState}}' | grep -qx active; then + sudo docker swarm init --advertise-addr "$(hostname -I | awk '{print $1}')" + fi + + echo "$GHCR_TOKEN" | sudo docker login ghcr.io -u "$GHCR_USER" --password-stdin + + echo "IMAGE_TAG in shell: [$IMAGE_TAG]" + + # For some reason, not pulling explicitly makes the docker stack deploy throw an error that it can't find the package. + sudo docker pull ghcr.io/knowledgefutures/platform:"$IMAGE_TAG" + + # deploy/update stack + sudo env IMAGE_TAG="$IMAGE_TAG" PREVIEW_HOST="$PREVIEW_HOST" PR_NUMBER="$PR_NUMBER" \ + docker stack deploy -c stack.preview.yml \ + --with-registry-auth --resolve-image always --prune "$STACK_NAME" + + # caddy's config is a bind mount, so swarm won't restart + # the proxy when only the Caddyfile changes on disk + sudo docker service update --force "${STACK_NAME}_proxy" + + sudo docker stack services "$STACK_NAME" + sudo docker image prune -f + + wait_rollout() { + echo "Beginning wait for rollout of $1..." + svc="$1" + timeout="${2:-600}" + end=$((SECONDS+timeout)) + + while (( SECONDS < end )); do + desired="$(sudo docker service inspect "$svc" --format '{{.Spec.Mode.Replicated.Replicas}}' 2>/dev/null || echo "")" + running="$(sudo docker service ps "$svc" --filter desired-state=running --format '{{.CurrentState}}' 2>/dev/null | grep -c '^Running' || true)" + state="$(sudo docker service inspect "$svc" --format '{{if .UpdateStatus}}{{.UpdateStatus.State}}{{end}}' 2>/dev/null || echo "")" + echo " $svc: desired=$desired running=$running state=$state" + + if [[ -n "$desired" && "$running" == "$desired" ]] && { [[ -z "$state" ]] || [[ "$state" == "completed" ]]; }; then + echo " $svc rollout complete" + return 0 + fi + + sleep 5 + done + + echo "Rollout timeout for $svc" + return 1 + } + + wait_rollout "${STACK_NAME}_platform" 600 + + EOS + + - name: Teardown preview stack + if: inputs.action == 'teardown' + env: + SSH_USER: ${{ secrets.SSH_USER }} + SSH_HOST: ${{ secrets.SSH_HOST_PREVIEW }} + STACK_NAME: ${{ steps.pr.outputs.stack_name }} + run: | + ssh "${SSH_USER}@${SSH_HOST}" \ + "env STACK_NAME='${STACK_NAME}' bash -s" <<'EOS' + set -euo pipefail + : "${STACK_NAME:?missing STACK_NAME}" + + echo "tearing down preview stack $STACK_NAME" + + if sudo docker stack ls --format '{{.Name}}' | grep -qx "$STACK_NAME"; then + sudo docker stack rm "$STACK_NAME" + sleep 10 + # prune volumes for this stack + sudo docker volume ls --filter "label=com.docker.stack.namespace=$STACK_NAME" -q \ + | xargs -r sudo docker volume rm || true + echo "stack $STACK_NAME removed" + else + echo "stack $STACK_NAME not found, nothing to tear down" + fi + + sudo docker image prune -f + + EOS + + - name: Comment on PR + if: inputs.action == 'deploy' + uses: actions/github-script@v7 + with: + script: | + const body = `Preview deployed at https://${{ steps.pr.outputs.host }}`; + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: ${{ steps.pr.outputs.number }}, + }); + const existing = comments.find(c => c.body.includes('Preview deployed at')); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: ${{ steps.pr.outputs.number }}, + body, + }); + } diff --git a/.github/workflows/pull-preview-script.sh b/.github/workflows/pull-preview-script.sh deleted file mode 100644 index 6af5557d58..0000000000 --- a/.github/workflows/pull-preview-script.sh +++ /dev/null @@ -1,7 +0,0 @@ -# install latest version of docker compose, by default it's using an ancient version -sudo curl -sL https://github.com/docker/compose/releases/latest/download/docker-compose-linux-"$(uname -m)" \ - -o "$(which docker-compose)" && sudo chmod +x "$(which docker-compose)" - -docker image prune -a -f - -df -h diff --git a/.github/workflows/pull-preview.yml b/.github/workflows/pull-preview.yml deleted file mode 100644 index 27a3dba4c0..0000000000 --- a/.github/workflows/pull-preview.yml +++ /dev/null @@ -1,93 +0,0 @@ -on: - workflow_call: - inputs: - PLATFORM_IMAGE: - required: true - type: string - JOBS_IMAGE: - required: true - type: string - MIGRATIONS_IMAGE: - required: true - type: string - SITE_BUILDER_IMAGE: - required: true - type: string - AWS_REGION: - required: true - type: string - ALWAYS_ON: - required: false - type: string - COMPOSE_FILES: - required: false - type: string - secrets: - AWS_ACCESS_KEY_ID: - required: true - AWS_SECRET_ACCESS_KEY: - required: true - GH_PAT_PR_PREVIEW_CLEANUP: - required: true - PREVIEW_DATACITE_REPOSITORY_ID: - required: true - PREVIEW_DATACITE_PASSWORD: - required: true - -permissions: - contents: read - deployments: write - pull-requests: write - statuses: write - -jobs: - preview: - timeout-minutes: 30 - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Copy .env file - run: cp ./self-host/.env.example ./self-host/.env - - - name: Configure pullpreview - env: - PLATFORM_IMAGE: ${{ inputs.PLATFORM_IMAGE }} - JOBS_IMAGE: ${{ inputs.JOBS_IMAGE }} - MIGRATIONS_IMAGE: ${{ inputs.MIGRATIONS_IMAGE }} - SITE_BUILDER_IMAGE: ${{ inputs.SITE_BUILDER_IMAGE }} - run: | - sed -i "s|image: PLATFORM_IMAGE|image: $PLATFORM_IMAGE|" docker-compose.preview.yml - sed -i "s|image: JOBS_IMAGE|image: $JOBS_IMAGE|" docker-compose.preview.yml - sed -i "s|image: MIGRATIONS_IMAGE|image: $MIGRATIONS_IMAGE|" docker-compose.preview.yml - sed -i "s|image: SITE_BUILDER_IMAGE|image: $SITE_BUILDER_IMAGE|" docker-compose.preview.yml - sed -i "s|DATACITE_REPOSITORY_ID: DATACITE_REPOSITORY_ID|DATACITE_REPOSITORY_ID: ${{ secrets.PREVIEW_DATACITE_REPOSITORY_ID }}|" docker-compose.preview.yml - sed -i "s|DATACITE_PASSWORD: DATACITE_PASSWORD|DATACITE_PASSWORD: ${{ secrets.PREVIEW_DATACITE_PASSWORD }}|" docker-compose.preview.yml - sed -i "s|email someone@example.com|email dev@pubpub.org|" self-host/caddy/Caddyfile - sed -i "s|example.com|{\$PUBLIC_URL}|" self-host/caddy/Caddyfile - - - name: Get ECR token - id: ecrtoken - run: echo "value=$(aws ecr get-login-password --region us-east-1)" >> $GITHUB_OUTPUT - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: "us-east-1" - - - uses: pullpreview/action@v5 - with: - label: preview - admins: 3mcd - compose_files: ./self-host/docker-compose.yml,docker-compose.preview.yml,${{ inputs.COMPOSE_FILES }} - default_port: 443 - instance_type: small - always_on: ${{ inputs.ALWAYS_ON }} - ports: 80,443 - registries: docker://AWS:${{steps.ecrtoken.outputs.value}}@246372085946.dkr.ecr.us-east-1.amazonaws.com - github_token: ${{ secrets.GH_PAT_PR_PREVIEW_CLEANUP }} - pre_script: "./.github/workflows/pull-preview-script.sh" - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ inputs.AWS_REGION }} - PULLPREVIEW_LOGGER_LEVEL: DEBUG diff --git a/.gitignore b/.gitignore index ccc4f294ca..72421cce93 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,10 @@ storybook-static ./playwright .local_data + +# infra decrypted env files (encrypted versions are tracked) +infra/.env +infra/.env.staging + +!.env*.enc +!.env.example \ No newline at end of file diff --git a/Caddyfile.test b/Caddyfile.test index 6f645d8b26..e46c20c054 100644 --- a/Caddyfile.test +++ b/Caddyfile.test @@ -24,7 +24,7 @@ example.com { # if you want to use a different domain for your files, you can do so here # for instance, now all your files will be accessible at assets.example.com -# if you go this route, be sure to update your ASSETS_STORAGE_ENDPOINT in .env and restart your services +# if you go this route, be sure to update your S3_ENDPOINT in .env and restart your services # assets.example.com { # reverse_proxy minio:9000 # } diff --git a/Dockerfile b/Dockerfile index 9b052300e4..a554830d7d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -136,5 +136,7 @@ COPY --from=withpackage --chown=node:node /usr/src/app/core/.next/static ./core/ COPY --from=withpackage --chown=node:node /usr/src/app/core/public ./core/public # needed to set the database url correctly based on PGHOST variables COPY --from=withpackage --chown=node:node /usr/src/app/core/.env.docker ./core/.env +# migration sql files, applied automatically during startup instrumentation +COPY --from=withpackage --chown=node:node /usr/src/app/core/prisma/migrations ./core/prisma/migrations -CMD ["node", "core/server.js"] +CMD ["node", "--enable-source-maps", "core/server.js"] diff --git a/core/instrumentation.ts b/core/instrumentation.ts index efaa830c1e..351dc481c3 100644 --- a/core/instrumentation.ts +++ b/core/instrumentation.ts @@ -16,13 +16,17 @@ export async function register() { } logger.info(`Registering instrumentation hook for ${process.env.NEXT_RUNTIME}`) + if (process.env.NEXT_RUNTIME === "nodejs") { + if (!process.env.SKIP_MIGRATIONS) { + const { runMigrations } = await import("./lib/server/migrate") + await runMigrations() + } + if (process.env.NODE_ENV === "development") { - logger.info( - "NEXT_RUNTIME is `nodejs` and NODE_ENV is `development`; skipping OTEL + Sentry registration." - ) return } + await import("./instrumentation.node.mts") } else { logger.info("NEXT_RUNTIME is not `nodejs`; skipping OTEL registration.") diff --git a/core/lib/editor/to-html.test.ts b/core/lib/editor/to-html.test.ts index 62885083d8..1574a17271 100644 --- a/core/lib/editor/to-html.test.ts +++ b/core/lib/editor/to-html.test.ts @@ -1,13 +1,12 @@ import { describe, expect, it } from "vitest" -// @ts-expect-error -import ponies from "../../prisma/seeds/ponies.snippet.html?raw" +import { poniesText } from "../../prisma/seeds/ponies.snippet" import { processEditorHTML } from "./process-editor-html" import { htmlToProsemirrorServer, prosemirrorToHTMLServer } from "./serialize-server" describe("renderNodeToHTML", () => { it("should be able to round trip a node and not lose any information", async () => { - const html = ponies + const html = poniesText expect(html).toBeDefined() diff --git a/core/lib/env/env.ts b/core/lib/env/env.ts index 9fb30ab208..16b8b0a8e1 100644 --- a/core/lib/env/env.ts +++ b/core/lib/env/env.ts @@ -16,20 +16,16 @@ export const env = createEnv({ server: { SELF_HOSTED: z.string().optional(), API_KEY: z.string(), - ASSETS_BUCKET_NAME: z.string(), - ASSETS_REGION: z.string(), - ASSETS_UPLOAD_KEY: z.string(), - ASSETS_UPLOAD_SECRET_KEY: z.string(), - ASSETS_STORAGE_ENDPOINT: z.string().url().optional(), - ASSETS_PUBLIC_ENDPOINT: z + S3_BUCKET_NAME: z.string(), + S3_REGION: z.string(), + S3_ACCESS_KEY: z.string(), + S3_SECRET_KEY: z.string(), + S3_ENDPOINT: z.string().url().optional(), + S3_PUBLIC_ENDPOINT: z .string() .url() .optional() - .transform((val) => - !val && process.env.ASSETS_STORAGE_ENDPOINT - ? process.env.ASSETS_STORAGE_ENDPOINT - : val - ), + .transform((val) => (!val && process.env.S3_ENDPOINT ? process.env.S3_ENDPOINT : val)), /** * Whether or not to verbosely log `memoize` cache hits and misses */ diff --git a/core/lib/server/assets.db.test.ts b/core/lib/server/assets.db.test.ts index 6aef731ca8..83ff57b9b6 100644 --- a/core/lib/server/assets.db.test.ts +++ b/core/lib/server/assets.db.test.ts @@ -11,13 +11,13 @@ const { getTrx } = createForEachMockedTransaction() beforeAll(async () => { // check if minio is up - if (!env.ASSETS_STORAGE_ENDPOINT) { + if (!env.S3_ENDPOINT) { throw new Error( "You should only run this test against a local minio instance, not to prod S3" ) } - const check = await fetch(env.ASSETS_STORAGE_ENDPOINT, { + const check = await fetch(env.S3_ENDPOINT, { method: "OPTIONS", }) diff --git a/core/lib/server/assets.ts b/core/lib/server/assets.ts index 06df71d6ea..5cbcf63763 100644 --- a/core/lib/server/assets.ts +++ b/core/lib/server/assets.ts @@ -63,13 +63,13 @@ export const generateMetadataFromS3 = async ( } export const getS3Client = () => { - const region = env.ASSETS_REGION - const key = env.ASSETS_UPLOAD_KEY - const secret = env.ASSETS_UPLOAD_SECRET_KEY + const region = env.S3_REGION + const key = env.S3_ACCESS_KEY + const secret = env.S3_SECRET_KEY logger.info({ msg: "Initializing S3 client", - endpoint: env.ASSETS_STORAGE_ENDPOINT, + endpoint: env.S3_ENDPOINT, region, key, secret, @@ -79,13 +79,13 @@ export const getS3Client = () => { } s3Client = new S3Client({ - endpoint: env.ASSETS_STORAGE_ENDPOINT, + endpoint: env.S3_ENDPOINT, region: region, credentials: { accessKeyId: key, secretAccessKey: secret, }, - forcePathStyle: !!env.ASSETS_STORAGE_ENDPOINT, // Required for MinIO + forcePathStyle: !!env.S3_ENDPOINT, // Required for MinIO }) logger.info({ @@ -99,10 +99,10 @@ export const getS3Client = () => { // this is bc, when using `minio` locally, the server // uses `minio:9000`, but for the client this does not make sense export const getPublicS3Client = () => { - const region = env.ASSETS_REGION - const key = env.ASSETS_UPLOAD_KEY - const secret = env.ASSETS_UPLOAD_SECRET_KEY - const publicEndpoint = env.ASSETS_PUBLIC_ENDPOINT || env.ASSETS_STORAGE_ENDPOINT + const region = env.S3_REGION + const key = env.S3_ACCESS_KEY + const secret = env.S3_SECRET_KEY + const publicEndpoint = env.S3_PUBLIC_ENDPOINT || env.S3_ENDPOINT return new S3Client({ endpoint: publicEndpoint, @@ -125,7 +125,7 @@ export const generateSignedAssetUploadUrl = async ( const client = getPublicS3Client() // use public client for signed URLs - const bucket = env.ASSETS_BUCKET_NAME + const bucket = env.S3_BUCKET_NAME const command = new PutObjectCommand({ Bucket: bucket, Key: key, @@ -140,7 +140,7 @@ export const generateSignedAssetUploadUrl = async ( const generateSignedUploadUrl = async (key: string) => { const client = getPublicS3Client() - const bucket = env.ASSETS_BUCKET_NAME + const bucket = env.S3_BUCKET_NAME const command = new PutObjectCommand({ Bucket: bucket, Key: key, @@ -175,9 +175,9 @@ export class InvalidFileUrlError extends Error { */ export const deleteFileFromS3 = async (fileUrl: string) => { const client = getPublicS3Client() - const bucket = env.ASSETS_BUCKET_NAME + const bucket = env.S3_BUCKET_NAME - const fileKey = fileUrl.split(new RegExp(`^.+${env.ASSETS_BUCKET_NAME}/`))[1] + const fileKey = fileUrl.split(new RegExp(`^.+${env.S3_BUCKET_NAME}/`))[1] if (!fileKey) { logger.error({ msg: "Unable to parse URL of uploaded file", fileUrl }) @@ -209,7 +209,7 @@ export const makeFileUploadPermanent = async ( }, trx = db ) => { - const matches = tempUrl.match(`(^.+${env.ASSETS_BUCKET_NAME}/)(temporary/.+)`) + const matches = tempUrl.match(`(^.+${env.S3_BUCKET_NAME}/)(temporary/.+)`) const prefix = matches?.[1] const source = matches?.[2] if (!source || !fileName || !prefix) { @@ -233,8 +233,8 @@ export const makeFileUploadPermanent = async ( }) const copyCommand = new CopyObjectCommand({ - CopySource: `${env.ASSETS_BUCKET_NAME}/${source}`, - Bucket: env.ASSETS_BUCKET_NAME, + CopySource: `${env.S3_BUCKET_NAME}/${source}`, + Bucket: env.S3_BUCKET_NAME, Key: newKey, }) @@ -256,7 +256,7 @@ export const makeFileUploadPermanent = async ( maxWaitTime: 10, minDelay: 1, }, - { Bucket: env.ASSETS_BUCKET_NAME, Key: newKey } + { Bucket: env.S3_BUCKET_NAME, Key: newKey } ) logger.debug({ msg: "successfully copied temp file to permanent directory", newKey, tempUrl }) await trx @@ -305,7 +305,7 @@ export const uploadFileToS3 = async ( } ): Promise => { const client = getS3Client() - const bucket = env.ASSETS_BUCKET_NAME + const bucket = env.S3_BUCKET_NAME const key = `${id}/${fileName}` const parallelUploads3 = new Upload({ diff --git a/core/lib/server/cache/autoCache.ts b/core/lib/server/cache/autoCache.ts index b6bad2699d..bea68ed00d 100644 --- a/core/lib/server/cache/autoCache.ts +++ b/core/lib/server/cache/autoCache.ts @@ -4,12 +4,14 @@ import type { AutoCacheOptions, DirectAutoOutput, ExecuteFn, SQB } from "./types import { cache } from "react" import { logger } from "logger" +import { tryCatch } from "utils/try-catch" import { env } from "~/lib/env/env" import { createCacheTag, createCommunityCacheTags } from "./cacheTags" import { getCommunitySlug } from "./getCommunitySlug" import { memoize } from "./memoize" import { cachedFindTables, directAutoOutput } from "./sharedAuto" +import { shouldSkipCache as shouldSkipCacheStore } from "./skipCacheStore" import { getTablesWithLinkedTables } from "./specialTables" import { getTransactionStore, setTransactionStore } from "./transactionStorage" @@ -60,10 +62,33 @@ const executeWithCache = < options?: AutoCacheOptions ) => { const executeFn = cache(async (...args: Parameters) => { - const communitySlug = options?.communitySlug ?? (await getCommunitySlug()) - const compiledQuery = qb.compile() + const willSkipCacheStore = shouldSkipCacheStore("store") + const willSkipCacheFn = options?.skipCacheFn?.() + + const willSkipCache = willSkipCacheStore || willSkipCacheFn + + if (willSkipCache) { + logger.debug( + willSkipCacheStore + ? `Skipping cache for query ${compiledQuery.sql} because of skipCacheStore` + : `Skipping cache for query ${compiledQuery.sql} because of skipCacheFn` + ) + + return qb[method](...args) as ReturnType + } + + const [error, communitySlug] = options?.communitySlug + ? [null, options.communitySlug] + : await tryCatch(getCommunitySlug()) + + if (error) { + logger.error(`Error getting community slug: ${error.message}`) + logger.error(compiledQuery.sql) + throw error + } + const tables = await cachedFindTables(compiledQuery, "select") const allTables = getTablesWithLinkedTables(tables) @@ -88,7 +113,7 @@ const executeWithCache = < asOne ) - if (shouldSkipCache || options?.skipCacheFn?.()) { + if (shouldSkipCache) { if (env.CACHE_LOG) { logger.debug(`AUTOCACHE: Skipping cache for query: ${asOne}`) } diff --git a/core/lib/server/cache/autoRevalidate.ts b/core/lib/server/cache/autoRevalidate.ts index bfcd8cf961..c89015ad12 100644 --- a/core/lib/server/cache/autoRevalidate.ts +++ b/core/lib/server/cache/autoRevalidate.ts @@ -4,11 +4,13 @@ import type { AutoRevalidateOptions, DirectAutoOutput, ExecuteFn, QB } from "./t import { revalidatePath, revalidateTag } from "next/cache" import { logger } from "logger" +import { tryCatch } from "utils/try-catch" import { env } from "~/lib/env/env" import { getCommunitySlug } from "./getCommunitySlug" import { revalidateTagsForCommunity } from "./revalidate" import { cachedFindTables, directAutoOutput } from "./sharedAuto" +import { shouldSkipCache as shouldSkipCacheStore } from "./skipCacheStore" import { setTransactionStore } from "./transactionStorage" const executeWithRevalidate = < @@ -20,11 +22,28 @@ const executeWithRevalidate = < options?: AutoRevalidateOptions ) => { const executeFn = async (...args: Parameters) => { - const communitySlug = options?.communitySlug ?? (await getCommunitySlug()) + const compiledQuery = qb.compile() - const communitySlugs = Array.isArray(communitySlug) ? communitySlug : [communitySlug] + const willSkipCacheStore = shouldSkipCacheStore("invalidate") - const compiledQuery = qb.compile() + if (willSkipCacheStore) { + logger.debug( + `Skipping revalidation for query ${compiledQuery.sql} because of skipCacheStore` + ) + return qb[method](...args) as ReturnType + } + + const [error, communitySlug] = options?.communitySlug + ? [null, options.communitySlug] + : await tryCatch(getCommunitySlug()) + + if (error) { + logger.error(`Error getting community slug: ${error.message}`) + logger.error(compiledQuery.sql) + throw error + } + + const communitySlugs = Array.isArray(communitySlug) ? communitySlug : [communitySlug] const tables = await cachedFindTables(compiledQuery, "mutation") diff --git a/core/lib/server/cache/skipCacheStore.ts b/core/lib/server/cache/skipCacheStore.ts new file mode 100644 index 0000000000..1d690270b9 --- /dev/null +++ b/core/lib/server/cache/skipCacheStore.ts @@ -0,0 +1,54 @@ +import { AsyncLocalStorage } from "node:async_hooks" + +import { logger } from "logger" + +const SKIP_CACHE_OPTIONS = ["store", "invalidate", "both"] as const +export type SkipCacheOptions = (typeof SKIP_CACHE_OPTIONS)[number] + +// tags +export const skipCacheStore = new AsyncLocalStorage<{ + /** + * Whether to store the result in the cache or invalidate it + */ + shouldSkipCache: "store" | "invalidate" | "both" | undefined +}>() + +export const setSkipCacheStore = ({ shouldSkipCache }: { shouldSkipCache: SkipCacheOptions }) => { + const store = skipCacheStore.getStore() + + if (!store) { + logger.debug("no skip cache store found") + return + } + + store.shouldSkipCache = shouldSkipCache + + return store +} + +/** + * whether or not to skip the cache + */ +export const shouldSkipCache = (skipCacheOptions: SkipCacheOptions) => { + const store = skipCacheStore.getStore() + + if (!store) { + return false + } + + if (store.shouldSkipCache === "both") { + return true + } + + return store.shouldSkipCache === skipCacheOptions +} + +/** + * wrap a function with this to skip storing and/or invalidating the cache + * useful when outside of community contexts and you don't want to cache results + */ +export const withUncached = (fn: () => Promise, skipCacheOptions?: SkipCacheOptions) => { + return skipCacheStore.run({ shouldSkipCache: skipCacheOptions ?? "invalidate" }, async () => { + return fn() + }) +} diff --git a/core/lib/server/migrate.ts b/core/lib/server/migrate.ts new file mode 100644 index 0000000000..4b2fa60db6 --- /dev/null +++ b/core/lib/server/migrate.ts @@ -0,0 +1,203 @@ +import type { Database } from "db/Database" +import type { PrismaMigrationsId } from "db/public" + +import { exec } from "node:child_process" +import { createHash, randomUUID } from "node:crypto" +import { existsSync, readdirSync, readFileSync } from "node:fs" +import { join, resolve } from "node:path" +import { promisify } from "node:util" +import { Kysely, PostgresDialect, sql } from "kysely" +import pg from "pg" + +import { tryCatch } from "utils/try-catch" + +const execAsync = promisify(exec) + +import { logger } from "logger" + +// arbitrary but stable id used to prevent concurrent migration runs across replicas +const ADVISORY_LOCK_ID = 72_398_241 + +const CREATE_MIGRATIONS_TABLE = ` +CREATE TABLE IF NOT EXISTS "_prisma_migrations" ( + "id" VARCHAR(36) NOT NULL, + "checksum" VARCHAR(64) NOT NULL, + "finished_at" TIMESTAMPTZ, + "migration_name" VARCHAR(255) NOT NULL, + "logs" TEXT, + "rolled_back_at" TIMESTAMPTZ, + "started_at" TIMESTAMPTZ NOT NULL DEFAULT now(), + "applied_steps_count" INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY ("id") +)` + +function sha256(content: string): string { + return createHash("sha256").update(content).digest("hex") +} + +async function waitForDatabase(pool: pg.Pool, maxAttempts = 30, intervalMs = 2000) { + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + const client = await pool.connect() + client.release() + return + } catch (err) { + if (attempt === maxAttempts) { + throw new Error( + `could not connect to database after ${maxAttempts} attempts: ${err}` + ) + } + + logger.info( + `database not ready, retrying in ${intervalMs}ms (${attempt}/${maxAttempts})...` + ) + await new Promise((r) => setTimeout(r, intervalMs)) + } + } +} + +export async function runMigrations() { + const connectionString = process.env.DATABASE_URL + if (!connectionString) { + throw new Error("DATABASE_URL is required to run migrations") + } + + // in next.js standalone mode, server.js does process.chdir(__dirname) + // which sets cwd to the app directory (e.g. /usr/src/app/core) + const migrationsDir = process.env.MIGRATIONS_DIR + ? resolve(process.env.MIGRATIONS_DIR) + : resolve(process.cwd(), "prisma", "migrations") + + if (!existsSync(migrationsDir)) { + logger.warn(`migrations directory not found at ${migrationsDir}, skipping`) + return + } + + const shouldReset = !!process.env.DB_RESET + const shouldSeed = !!process.env.DB_SEED + + logger.info(`running migrations from ${migrationsDir}`) + + // max: 1 ensures every operation (kysely typed queries + raw pool.query) + // shares the same underlying connection session, keeping the advisory lock valid + const pool = new pg.Pool({ connectionString, max: 1 }) + const db = new Kysely({ dialect: new PostgresDialect({ pool }) }) + + try { + await waitForDatabase(pool) + + await sql`SELECT pg_advisory_lock(${sql.lit(ADVISORY_LOCK_ID)})`.execute(db) + + if (shouldReset) { + logger.info("resetting database (DB_RESET is set)") + await pool.query("DROP SCHEMA public CASCADE") + await pool.query("CREATE SCHEMA public") + } + + // raw string query so pg uses the simple protocol (supports multi-statement sql) + await pool.query(CREATE_MIGRATIONS_TABLE) + + const failed = await db + .selectFrom("_prisma_migrations") + .select("migration_name") + .where("finished_at", "is", null) + .where("rolled_back_at", "is", null) + .execute() + + if (failed.length > 0) { + const names = failed.map((r) => r.migration_name).join(", ") + throw new Error( + `found migrations in a failed state that need manual resolution: ${names}. ` + + `mark them as rolled back or delete their rows from _prisma_migrations to proceed.` + ) + } + + const applied = await db + .selectFrom("_prisma_migrations") + .select("migration_name") + .where("finished_at", "is not", null) + .where("rolled_back_at", "is", null) + .execute() + + const appliedNames = new Set(applied.map((r) => r.migration_name)) + + const dirs = readdirSync(migrationsDir, { withFileTypes: true }) + .filter((d) => d.isDirectory()) + .map((d) => d.name) + .sort() + + let count = 0 + + for (const dir of dirs) { + if (appliedNames.has(dir)) { + continue + } + + const sqlPath = join(migrationsDir, dir, "migration.sql") + if (!existsSync(sqlPath)) { + continue + } + + const migrationSql = readFileSync(sqlPath, "utf-8") + const checksum = sha256(migrationSql) + const id = randomUUID() as PrismaMigrationsId + + logger.info(`applying migration: ${dir}`) + + await db + .insertInto("_prisma_migrations") + .values({ id, checksum, migration_name: dir }) + .execute() + + try { + await pool.query(migrationSql) + } catch (err) { + await db + .updateTable("_prisma_migrations") + .set({ logs: String(err) }) + .where("id", "=", id) + .execute() + throw err + } + + await db + .updateTable("_prisma_migrations") + .set({ finished_at: new Date(), applied_steps_count: 1 }) + .where("id", "=", id) + .execute() + + count++ + } + + if (count > 0) { + logger.info(`applied ${count} migration(s)`) + } else { + logger.info("database is up to date, no pending migrations") + } + + if (shouldSeed) { + logger.info("running database seed (DB_SEED is set)") + const { seed } = await import("~/prisma/seed") + + // prevents autocache from running, breaking seed + const { withUncached } = await import("~/lib/server/cache/skipCacheStore") + await withUncached(seed, "both") + + // try and reset cache + logger.info(`Clearing cache...`) + const [error, output] = await tryCatch( + execAsync("echo 'FLUSHALL' | nc $VALKEY_HOST 6379") + ) + + if (error || output.stderr) { + logger.error(`Error clearing cache: ${error?.message || output?.stderr}`) + } else { + logger.info(`Cache cleared: ${output.stdout}`) + } + } + + await sql`SELECT pg_advisory_unlock(${sql.lit(ADVISORY_LOCK_ID)})`.execute(db) + } finally { + await db.destroy() + } +} diff --git a/core/prisma/seed.ts b/core/prisma/seed.ts index d125f071c8..ad5337af06 100644 --- a/core/prisma/seed.ts +++ b/core/prisma/seed.ts @@ -16,11 +16,7 @@ const starterId = "bbbbbbbb-bbbb-4bbb-bbbb-bbbbbbbbbbbb" as CommunitiesId const blankId = "cccccccc-cccc-4ccc-cccc-cccccccccccc" as CommunitiesId const coarNotifyId = "dddddddd-dddd-4ddd-dddd-dddddddddddd" as CommunitiesId -async function main() { - // do not seed arcadia if the minimal seed flag is set - // this is because it will slow down ci/testing - // this flag is set in the `globalSetup.ts` file - // and in e2e.yml +export async function seed() { // eslint-disable-next-line no-restricted-properties const shouldSeedLegacy = !process.env.MINIMAL_SEED @@ -53,16 +49,22 @@ async function main() { await seedCoarNotify(coarNotifyId) } -main() - .then(async () => { - logger.info("Finished seeding, exiting...") - process.exit(0) - }) - .catch(async (e) => { - if (!isUniqueConstraintError(e)) { + +// cli entrypoint: only auto-run when executed directly as a script +const isCli = process.argv[1]?.endsWith("seed.ts") || process.argv[1]?.endsWith("seed.js") + +if (isCli) { + seed() + .then(async () => { + logger.info("Finished seeding, exiting...") + process.exit(0) + }) + .catch(async (e) => { + if (!isUniqueConstraintError(e)) { + logger.error(e) + process.exit(1) + } logger.error(e) - process.exit(1) - } - logger.error(e) - logger.info("Attempted to add duplicate entries, db is already seeded?") - }) + logger.info("Attempted to add duplicate entries, db is already seeded?") + }) +} diff --git a/core/prisma/seed/seedCommunity.ts b/core/prisma/seed/seedCommunity.ts index 28ba880171..e4872c743e 100644 --- a/core/prisma/seed/seedCommunity.ts +++ b/core/prisma/seed/seedCommunity.ts @@ -1196,10 +1196,12 @@ export async function seedCommunity< logger.info( `${createdCommunity.name}: ${options?.parallelPubs ? "Parallelly" : "Sequentially"} - Creating ${createPubRecursiveInput.length} pubs` ) + if (options?.parallelPubs) { const input = createPubRecursiveInput.map((input) => createPubRecursiveNew({ ...input })) - setInterval(() => { + // using will auto clear the interval when the block is exited + using _interval = setInterval(() => { logger.info(`${createdCommunity.name}: Creating Pubs...`) }, 1000) diff --git a/core/prisma/seeds/legacy.ts b/core/prisma/seeds/legacy.ts index 7a3468c2cc..9766a0f636 100644 --- a/core/prisma/seeds/legacy.ts +++ b/core/prisma/seeds/legacy.ts @@ -1,6 +1,5 @@ import type { CommunitiesId, PubsId } from "db/public" -import { readFile } from "node:fs/promises" import { faker } from "@faker-js/faker" import { CoreSchemaType, MemberRole } from "db/public" @@ -13,7 +12,7 @@ import { usersExisting } from "./users" const abstract = `

The development of AAV capsids for therapeutic gene delivery has exploded in popularity over the past few years. However, humans aren’t the first or only species using viral capsids for gene delivery — wasps evolved this tactic over 100 million years ago. Parasitoid wasps that lay eggs inside arthropod hosts have co-opted ancient viruses for gene delivery to manipulate multiple aspects of the host’s biology, thereby increasing the probability of survival of the wasp larvae

` export const seedLegacy = async (communityId?: CommunitiesId) => { - const poniesText = await readFile(new URL("./ponies.snippet.html", import.meta.url), "utf-8") + const { poniesText } = await import("./ponies.snippet") const articleSeed = (number = 1_000, asRelation = false) => Array.from({ length: number }, (_, idx) => { diff --git a/core/prisma/seeds/ponies.snippet.html b/core/prisma/seeds/ponies.snippet.ts similarity index 96% rename from core/prisma/seeds/ponies.snippet.html rename to core/prisma/seeds/ponies.snippet.ts index 67faad2418..827150c924 100644 --- a/core/prisma/seeds/ponies.snippet.html +++ b/core/prisma/seeds/ponies.snippet.ts @@ -1,3 +1,5 @@ +// prettier-ignore +export const poniesText = `
Experimental design >166±29kg166 \pm 29 kg166 pm 29 kgFabrication of construct >=10μm= 10 \mu m= 10 mu mFabrication of construct >300x300μm300x300 \mu m300x300 mu mFabrication of construct >1300μm1300 \mu m1300 mu mFabrication of construct >c−115 mm\cdot sec^{-1}15 mmcdot sec^{-1}Fabrication of construct >22−24°C22 - 24 \degree \text{C}22 - 24 degree \text{C}Fabrication of construct >30−50%30 - 50\%30 - 50%Fabrication of construct >70%70\%70%Fabrication of construct >l−12.2 g\cdot ml^{-1}2.2 gcdot ml^{-1}Fabrication of construct >α\alphaalphaFabrication of construct >=3.83μm= 3.83 \mu m= 3.83 mu mFabrication of construct >l−10.13 g\cdot ml^{-1}0.13 gcdot ml^{-1}Fabrication of construct >v−140\% w\cdot v^{-1}40% wcdot v^{-1}Fabrication of construct >α\alphaalphaFabrication of construct >0.22μm0.22 \mu m0.22 mu mFabrication of construct >4°C4\degree \text{C}4degree \text{C}Fabrication of construct >=250μm= 250 \mu m= 250 mu mFabrication of construct >700μm700 \mu m700 mu mFabrication of construct >70%70\%70%In Vitro pre-culture >37°C37\degree \text{C}37degree \text{C}In Vitro pre-culture >v−10.2\% w\cdot v^{-1}0.2% wcdot v^{-1}In Vitro pre-culture >v−10.075\% w\cdot v^{-1}0.075% wcdot v^{-1}In Vitro pre-culture >v−110\% v\cdot v^{-1}10% vcdot v^{-1}In Vitro pre-culture >1%1\%1%In Vitro pre-culture >L−1100 U\cdot mL^{-1}100 Ucdot mL^{-1}In Vitro pre-culture >L−1100 \mu g\cdot mL^{-1}100 mu gcdot mL^{-1}In Vitro pre-culture >100μl100 \mu l100 mu lIn Vitro pre-culture >l−1100 ng\cdot ml^{-1}100 ngcdot ml^{-1}Surgical procedure >g−110 \mu g\cdot kg^{-1}10 mu gcdot kg^{-1}Surgical procedure >g−10.1 mg\cdot kg^{-1}0.1 mgcdot kg^{-1}Surgical procedure >g−10.06 mg\cdot kg^{-1}0.06 mgcdot kg^{-1}Surgical procedure >g−12.2 mg\cdot kg^{-1}2.2 mgcdot kg^{-1}Surgical procedure >g−110 \mu g\cdot kg^{-1}10 mu gcdot kg^{-1}Surgical procedure >g−10.5 mg\cdot kg^{-1}0.5 mgcdot kg^{-1}Surgical procedure >g−10.6 mg\cdot kg^{-1}0.6 mgcdot kg^{-1}Surgical procedure >g−10.1 - 0.2 mg\cdot kg^{-1}0.1 - 0.2 mgcdot kg^{-1}Surgical procedure >g−110 - 15 mg\cdot kg^{-1}10 - 15 mgcdot kg^{-1}Surgical procedure >g−120 mg\cdot kg^{-1}20 mgcdot kg^{-1}Surgical procedure >g−10.6 mg\cdot kg^{-1}0.6 mgcdot kg^{-1}Surgical procedure >g−15 mg\cdot kg^{-1}5 mgcdot kg^{-1}Euthanasia and sample harvest >g−10.06mg\cdot kg^{-1}0.06mgcdot kg^{-1}Euthanasia and sample harvest >g−12.2 mg\cdot kg^{-1}2.2 mgcdot kg^{-1}Euthanasia and sample harvest >g−11400 mg\cdot kg^{-1}1400 mgcdot kg^{-1}Biomechanical evaluation >n−10.250 N\cdot min^{-1}0.250 Ncdot min^{-1}Biomechanical evaluation >200μm200 \mu m200 mu mBiomechanical evaluation >10−12%10-12 \%10-12 %Biochemical evaluation >60°C60\degree \text{C}60degree \text{C}Biochemical evaluation >−80°C-80\degree \text{C}-80degree \text{C}Microcomputed tomography >=200μA= 200 \mu A= 200 mu AMicrocomputed tomography >=30μm3= 30 \mu m^{3}= 30 mu m^{3}Histological evaluation >4%4\%4%Histological evaluation >5μm5 \mu m5 mu mHistological evaluation >l−11.083 mg\cdot ml^{-1}1.083 mgcdot ml^{-1}Histological evaluation >l−10.06 mg\cdot ml^{-1}0.06 mgcdot ml^{-1}Histological evaluation >4%4\%4%Histological evaluation >5μm5 \mu m5 mu mStatistical analysis >±\pmpmIn vitro >⋅\cdotcdotIn vitro >g−1199.7 \pm 67.7 \mu g\cdot \mu g^{-1}199.7 pm 67.7 mu gcdot mu g^{-1}In vitro >g−13702 \pm 2111 U\cdot\mu g^{-1}3702 pm 2111 Ucdotmu g^{-1} >g−130.46 \pm 15.95 \mu g\cdot\mu g^{-1}30.46 pm 15.95 mu gcdotmu g^{-1} >g−124.44 \pm 15.31 \mu g\cdot g^{-1}24.44 pm 15.31 mu gcdot g^{-1} >g−179.66 \pm 91.21 \mu g\cdot\mu g^{-1}79.66 pm 91.21 mu gcdotmu g^{-1} >g−1134.21\pm 153.73 \mu g\cdot\mu g^{-1}134.21pm 153.73 mu gcdotmu g^{-1}0.31 \pm 0.13 MPa0.31 pm 0.13 MPa0.42 \pm 0.19 MPa0.42 pm 0.19 MPa1.75 \pm 0.80 MPa1.75 pm 0.80 MPa2.22 \pm 0.48 MPa2.22 pm 0.48 MPa1.86 \pm 0.78 MPa1.86 pm 0.78 MPa2.19 \pm 0.77 MPa2.19 pm 0.77 MPa >6.14%±10.09%6.14\% \pm 10.09\%6.14% pm 10.09% >4.73%±4.93%4.73\% \pm 4.93\%4.73% pm 4.93% >81.38%±15.37%81.38\% \pm 15.37\%81.38% pm 15.37% >74.71%±12.44%74.71\% \pm 12.44\%74.71% pm 12.44% >12.48%±9.75%12.48\% \pm 9.75\%12.48% pm 9.75% >20.56%±10.54%20.56\% \pm 10.54\%20.56% pm 10.54% >79.02±16.18%79.02 \pm 16.18 \%79.02 pm 16.18 % >63.20±13.90%63.20 \pm 13.90 \%63.20 pm 13.90 %Supplementary Table 2. Lymphocyte.
+` diff --git a/dev.Caddyfile b/dev.Caddyfile index f8ece58b87..21f1093e76 100644 --- a/dev.Caddyfile +++ b/dev.Caddyfile @@ -45,13 +45,13 @@ # visit: http://localhost:8080/my-community/journal-2024/ handle_path /sites/* { - import s3site {$ASSETS_BUCKET_NAME:assets} {$S3_ENDPOINT:garage:3900} + import s3site {$S3_BUCKET_NAME:assets} {$S3_ENDPOINT:garage:3900} } # simpler path without /sites prefix # handles /{communitySlug}/{subpath}/... handle /* { - import s3site "{$ASSETS_BUCKET_NAME:assets}/sites" {$S3_ENDPOINT:garage:3900} + import s3site "{$S3_BUCKET_NAME:assets}/sites" {$S3_ENDPOINT:garage:3900} } } @@ -72,7 +72,7 @@ # # { # filesystem sites s3 { -# bucket {$ASSETS_BUCKET_NAME:assets} +# bucket {$S3_BUCKET_NAME:assets} # region {$S3_REGION:garage} # endpoint {$S3_ENDPOINT:http://garage:3900} # use_path_style diff --git a/development/docker-compose.dev.yml b/development/docker-compose.dev.yml index eb6607257c..497ae24538 100644 --- a/development/docker-compose.dev.yml +++ b/development/docker-compose.dev.yml @@ -86,8 +86,8 @@ services: volumes: - ../dev.Caddyfile:/etc/caddy/Caddyfile environment: - - ASSETS_BUCKET_NAME=${ASSETS_BUCKET_NAME:-assets.v7.pubpub.org} - - S3_REGION=${ASSETS_REGION:-garage} + - S3_BUCKET_NAME=${S3_BUCKET_NAME:-assets.v7.pubpub.org} + - S3_REGION=${S3_REGION:-garage} - S3_ENDPOINT=garage:3900 ports: - "8080:8080" diff --git a/docker-compose.base.yml b/docker-compose.base.yml index 5a61122955..0a9be47737 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -32,10 +32,10 @@ services: entrypoint: > /bin/sh -c ' /usr/bin/mc alias set myminio http://minio:9000 "$${MINIO_ROOT_USER}" "$${MINIO_ROOT_PASSWORD}"; - /usr/bin/mc mb --ignore-existing myminio/"$${ASSETS_BUCKET_NAME}"; - /usr/bin/mc anonymous set download myminio/"$${ASSETS_BUCKET_NAME}"; - /usr/bin/mc admin user add myminio "$${ASSETS_UPLOAD_KEY}" "$${ASSETS_UPLOAD_SECRET_KEY}"; - /usr/bin/mc admin policy attach myminio readwrite --user "$${ASSETS_UPLOAD_KEY}";' + /usr/bin/mc mb --ignore-existing myminio/"$${S3_BUCKET_NAME}"; + /usr/bin/mc anonymous set download myminio/"$${S3_BUCKET_NAME}"; + /usr/bin/mc admin user add myminio "$${S3_ACCESS_KEY}" "$${S3_SECRET_KEY}"; + /usr/bin/mc admin policy attach myminio readwrite --user "$${S3_ACCESS_KEY}";' db: image: postgres:15 diff --git a/docker-compose.preview.pr.yml b/docker-compose.preview.pr.yml deleted file mode 100644 index c1e1c23609..0000000000 --- a/docker-compose.preview.pr.yml +++ /dev/null @@ -1,7 +0,0 @@ -services: - platform: - environment: - PUBPUB_URL: ${PULLPREVIEW_URL} - caddy: - environment: - PUBLIC_URL: ${PULLPREVIEW_PUBLIC_DNS} diff --git a/docker-compose.preview.sandbox.yml b/docker-compose.preview.sandbox.yml deleted file mode 100644 index 9c9123799c..0000000000 --- a/docker-compose.preview.sandbox.yml +++ /dev/null @@ -1,7 +0,0 @@ -services: - platform: - environment: - PUBPUB_URL: https://sandbox.pubpub.org - caddy: - environment: - PUBLIC_URL: sandbox.pubpub.org diff --git a/docker-compose.preview.yml b/docker-compose.preview.yml deleted file mode 100644 index 6a4f431ca9..0000000000 --- a/docker-compose.preview.yml +++ /dev/null @@ -1,82 +0,0 @@ -services: - platform: - image: PLATFORM_IMAGE - environment: - POSTGRES_USER: preview - POSTGRES_PASSWORD: preview - POSTGRES_DB: preview - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin - ASSETS_UPLOAD_KEY: preview-different - ASSETS_UPLOAD_SECRET_KEY: preview-different123 - ASSETS_STORAGE_ENDPOINT: https://${PULLPREVIEW_PUBLIC_DNS}/a - FLAGS: uploads:off,invites:off,disabled-actions:http+email - ENV_NAME: sandbox - DATACITE_API_URL: https://api.test.datacite.org - DATACITE_REPOSITORY_ID: DATACITE_REPOSITORY_ID - DATACITE_PASSWORD: DATACITE_PASSWORD - - minio-init: - restart: on-failure - environment: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin - ASSETS_UPLOAD_KEY: preview-different - ASSETS_UPLOAD_SECRET_KEY: preview-different123 - - minio: - environment: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin - MINIO_BROWSER_REDIRECT_URL: https://${PULLPREVIEW_PUBLIC_DNS}/assets-ui - # volumes: - # - ./minio:/data - platform-jobs: - image: JOBS_IMAGE - platform-migrations: - image: MIGRATIONS_IMAGE - restart: on-failure - command: pnpm --filter core reset - site-builder: - image: SITE_BUILDER_IMAGE - depends_on: - - platform - - platform-jobs - - minio - ports: - - "4000:4000" - restart: on-failure - networks: - - app-network - environment: - - PUBPUB_URL=http://platform:3000 - - S3_ENDPOINT=http://minio:9000 - - S3_REGION=us-east-1 - - S3_ACCESS_KEY=preview-different - - S3_SECRET_KEY=preview-different123 - - S3_BUCKET_NAME=assets - - PORT=4000 - - SITES_BASE_URL=https://${PULLPREVIEW_PUBLIC_DNS}/sites - - caddy: - image: CADDY_SITES_IMAGE - restart: on-failure - depends_on: - - platform - - platform-jobs - - minio - env_file: .env - environment: - - S3_ENDPOINT=http://minio:9000 - - S3_REGION=us-east-1 - - ASSETS_BUCKET_NAME=assets - - ASSETS_UPLOAD_KEY=preview-different - - ASSETS_UPLOAD_SECRET_KEY=preview-different123 - ports: - - "443:443" - volumes: - - ./caddy:/etc/caddy - - caddy-data:/data - - caddy-config:/config - networks: - - app-network diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 5e7cc46fc8..f00f4f8242 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -130,11 +130,11 @@ services: - integration environment: - PUBPUB_URL=http://integration-tests:3000 - - S3_ENDPOINT=${ASSETS_STORAGE_ENDPOINT:-http://minio:9000} - - S3_REGION=${ASSETS_REGION:-us-east-1} - - S3_ACCESS_KEY=${ASSETS_UPLOAD_KEY:-preview-different} - - S3_SECRET_KEY=${ASSETS_UPLOAD_SECRET_KEY:-preview-different123} - - S3_BUCKET_NAME=${ASSETS_BUCKET_NAME:-byron} + - S3_ENDPOINT=${S3_ENDPOINT:-http://minio:9000} + - S3_REGION=${S3_REGION:-us-east-1} + - S3_ACCESS_KEY=${S3_ACCESS_KEY:-preview-different} + - S3_SECRET_KEY=${S3_SECRET_KEY:-preview-different123} + - S3_BUCKET_NAME=${S3_BUCKET_NAME:-byron} - PORT=4000 volumes: diff --git a/infra/.env.example b/infra/.env.example new file mode 100644 index 0000000000..b90d0eb5d7 --- /dev/null +++ b/infra/.env.example @@ -0,0 +1,46 @@ +# production environment variables +# copy this to .env, fill in real values, then encrypt with: +# sops -e --input-type dotenv --output-type dotenv .env > .env.enc + +PUBPUB_HOSTNAME=app.pubpub.org +PUBPUB_URL=https://app.pubpub.org + +POSTGRES_USER= +POSTGRES_PASSWORD= +POSTGRES_DB=pubpub +DATABASE_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB} + +PGHOST=db +PGPORT=5432 +PGUSER=${POSTGRES_USER} +PGPASSWORD=${POSTGRES_PASSWORD} +PGDATABASE=${POSTGRES_DB} + +VALKEY_HOST=cache + +MINIO_ROOT_USER= +MINIO_ROOT_PASSWORD= +S3_BUCKET_NAME=assets +S3_ACCESS_KEY= +S3_SECRET_KEY= +S3_REGION=us-east-1 +S3_ENDPOINT=http://minio:9000 +S3_ENDPOINT=https://app.pubpub.org/a + +S3_ENDPOINT=http://minio:9000 +S3_REGION=us-east-1 + +SITE_BUILDER_ENDPOINT=http://site-builder:4000 +SITE_BUILDER_API_KEY= + +JWT_SECRET= +MAILGUN_SMTP_HOST= +MAILGUN_SMTP_PORT=587 +MAILGUN_SMTP_PASSWORD= +MAILGUN_SMTP_USERNAME= +GCLOUD_KEY_FILE=xxx + +OTEL_SERVICE_NAME=pubpub-v7-prod +HONEYCOMB_API_KEY= + +API_KEY= diff --git a/infra/.env.preview.enc b/infra/.env.preview.enc new file mode 100644 index 0000000000..2a40937253 --- /dev/null +++ b/infra/.env.preview.enc @@ -0,0 +1,51 @@ +#ENC[AES256_GCM,data:01FuyTOAi8hcwTlE3h5bzuxLnxIcAUUOaBWOgewecUak,iv:wlTDb983ry3QmHPTNO831TCRc/y1rNfVBdEBltsjQHk=,tag:Jx4ihkq9zJe5DrhKh3VzUg==,type:comment] +#ENC[AES256_GCM,data:F8lt5UbzAe/bl9JjbNo9PMP/XbpxNDqubPdlBIy5G7FlBXHKAs4kMOUrcG749TP7QfdjRbUVxGWTeRE=,iv:VCsmfGyoATmwB7ZA7q+cE/5hAdvhctqUR3V36uyvugc=,tag:+iAPVAkvIv5mBoKmyAcRXg==,type:comment] +#ENC[AES256_GCM,data:SfJkrX4rrOWnflZeLZB+OTo5ZEjmoa+hGALSh6OXyWhnMwYuWgj4imsw/Pt06BnD6mULcUX6w/WSbxiChpK6Rz2Uow==,iv:fxo360dsbqLLuyQRoxnbgLsP0XFurYGwCV7Q2mule/o=,tag:96vqOoeBrInNq0iAvHygFA==,type:comment] +#ENC[AES256_GCM,data:S1LlGLvEHPWvhx4deVBIM4LlBQCRLoyZBpwhLqqabmUHJmg5ZU4u7h15VZdxulQUCA==,iv:HB1P2F3SbjlG6WfU/M03X/Ix0iZQhvZlTCmQG+9Dm+k=,tag:fYGwJb5upU4BbHTFn5sXDA==,type:comment] +#ENC[AES256_GCM,data:18ixZWR90g7X5C3/wu/lG8mCRS5fvfe5PrCThAwBqCqSpYmnCmw3kX9NtKHVTwt1Po0mkw==,iv:hxsNw3R9ionzxa70IjaXLyk++KfW6lCEfCI8yot3Ecg=,tag:61sgT8b7SWWlW7u/hAbbUQ==,type:comment] +POSTGRES_USER=ENC[AES256_GCM,data:NykRLilL23I=,iv:uSUgclbI+u5HlqSazcXTdQMF3+6MPnF5gBqhgFrdxQ8=,tag:XFNiQkZRzfClqyzaPDqGGg==,type:str] +POSTGRES_PASSWORD=ENC[AES256_GCM,data:cjdQF1aMDBA=,iv:tkZMn6+06SKK95jTNmuTsdrWcuB3vQfTkOYaIKTwpCc=,tag:V09wYVU5J+mtkCrmQ9+2Mw==,type:str] +POSTGRES_DB=ENC[AES256_GCM,data:euB33ubvYg==,iv:eq7IpQpUOqO4RGLmR+i93W1Th9ldQxO+s6UOKMrP+do=,tag:K//krGc7ryYjXqdJoAZu/w==,type:str] +DATABASE_URL=ENC[AES256_GCM,data:0lZeeWD8t16+wVoVx3GG5avuecZvJtEjfRq3r3DldeNQq66C6umx/s84LcdoVA==,iv:pBG3RlFFjyUSgCEVVMesLd21GM/ChoPw6Kw5ZY9ErII=,tag:aHv0zEgRyjUO/HyjkrIShg==,type:str] +PGHOST=ENC[AES256_GCM,data:ti8=,iv:Y+g2zJxw6AZnm3JXLDuUJFtOkPUMiyTFCsm+0GoA1VI=,tag:2jp/X+VV7SNVbMUmC1e7xA==,type:str] +PGPORT=ENC[AES256_GCM,data:2RhQ/g==,iv:8rikqp+jWc6t8cJLLuLTrGhetFI7wu+LLUxI2QMCMAI=,tag:HtqOSp2JKUAZkHq6m6KDoQ==,type:str] +PGUSER=ENC[AES256_GCM,data:rjuJ2J89/Dw=,iv:BAK/cCaQYwyJiMV+pAVTRToBF5s12/A8WXTxT9RRjN4=,tag:n4hBU76ZReuJdNpCs+cowA==,type:str] +PGPASSWORD=ENC[AES256_GCM,data:EQHAp8G72pU=,iv:xKbh12AcnOBUcw+swC3McPYb23nyL12RV9iDt8qKBj0=,tag:IahOy866FQBSqaIvFCbq3w==,type:str] +PGDATABASE=ENC[AES256_GCM,data:nYwXkfTO+w==,iv:qq86WLG0XxFiwiQuhGiQXduAkqUjSzEgLPxJTnc02V0=,tag:VVf9bzQtqR5O56X+yMtyeQ==,type:str] +VALKEY_HOST=ENC[AES256_GCM,data:OlI8Ois=,iv:apN+8DyufGyGHh/+fGJG/4Zt1SSLHr8h9IJzWAVcc5g=,tag:mJoNwGnuv/yGygA4SE81Gw==,type:str] +MINIO_ROOT_USER=ENC[AES256_GCM,data:R1xiVwU4mHU8le4zaA==,iv:hQE2tkfqR0x2WisPx+g89YtPC1+YX4LkHyZ608OxzI8=,tag:fYJvFtC/vwW4hfFj2k32Iw==,type:str] +MINIO_ROOT_PASSWORD=ENC[AES256_GCM,data:JJoqNeEK/bEFlsP4xQ==,iv:pdBqmcJ+Pxhv3ZrIs5DTQSwHmsfjqhlbpCkO1Nv/OZk=,tag:sWZ0GxJfxQHYE51SFtghxA==,type:str] +S3_BUCKET_NAME=ENC[AES256_GCM,data:33QLz1weIvl27fkuz0M=,iv:4QWmPSPE2YpYP6dGc3hNzHq4EW2vRRANG0Sn/1OSQLE=,tag:0Tqwb5DvhwprvXeNw4YNCw==,type:str] +S3_ACCESS_KEY=ENC[AES256_GCM,data:GtYFUF+JbS57Nmw=,iv:AxwCMu9e4yJ/tiemM+Wyk6QqYdzQ3oZAjicDMKVEXWA=,tag:hXplo3UDLXy0suWJ2KGM+w==,type:str] +S3_SECRET_KEY=ENC[AES256_GCM,data:qV3HZp7Qem7u2+Q=,iv:Dwat6yRxAFMu2C1Lhl1J6K+iZ2X6/n9jmdjlz2N7AK4=,tag:YreDcISrsSTMpUPcGsHvaA==,type:str] +S3_REGION=ENC[AES256_GCM,data:liRqnsqwJ0Ct,iv:XUfhFifl/MZWf9Yy/02Bhjvlp4jBuQEML29dg+18SvY=,tag:ICTYnLtmRbVVjyYAK3nTnw==,type:str] +S3_ENDPOINT=ENC[AES256_GCM,data:jzzGDkgL33OjHDCn/JrCeac=,iv:4xyA52QskQeXZLaMWqAF8h+JYiUKNdoaBqoWknxLe0U=,tag:pVdb89AY/ZEirwQTpOwvLQ==,type:str] +#ENC[AES256_GCM,data:ZvethfQn3G93hzb9HylzPPV+SNHU1dLdUWddhZKfKYDXR48m4/pxI0Bfem2xEuG6N5vI756KFzhnjwDqEyk/zzKO1g==,iv:gqU2h8tctMH8UEixoGeze/u1osGk2HjPIn6KJkA7oss=,tag:bba0OSt+ukemN2PAA+m1mA==,type:comment] +SITE_BUILDER_ENDPOINT=ENC[AES256_GCM,data:P3Aohsj9hrSbbEB4cIZH5e4Dwic4i131,iv:NRSCdyFm+LcaoSEoin27cFkIQEYVHUZcnyKtHChlDn8=,tag:5jQP0xeUwsS8YU3fi3oNRw==,type:str] +SITE_BUILDER_API_KEY=ENC[AES256_GCM,data:h/Tt,iv:OHa3LPP4HAC+/6ynmQyqu/4kIOW5jTBhPLmmjBAXerA=,tag:UAUvhlNI9t/gMcG0bZAfzA==,type:str] +MAILGUN_SMTP_HOST=ENC[AES256_GCM,data:il5J,iv:2qhdXyt8M9Waj2Blp3/bZ6Lbd1CSnL44ni0w9BKMk2M=,tag:6L7LrjsV5dzd4SJ2CAOwbA==,type:str] +MAILGUN_SMTP_PORT=ENC[AES256_GCM,data:Cq0A,iv:HyZccNohY3LOBL83imGrKJRXaI5HNnOAbfkf7jsgUUU=,tag:onTdFiHQVfmMq38ffw7QLA==,type:str] +MAILGUN_SMTP_PASSWORD=ENC[AES256_GCM,data:Ux3L,iv:b72bNCwUdbI/ncfm0mozrkgnF7vPsBTFFWydptlh10w=,tag:HzbugC9DzQfnym0y6O91fg==,type:str] +MAILGUN_SMTP_USERNAME=ENC[AES256_GCM,data:u/w/,iv:+8s/rhemnPmmFFKg8HiB5KFT/kHfVwXrh6F7NKVBdwA=,tag:EZ9veppRkG4miHNCDKzETA==,type:str] +GCLOUD_KEY_FILE=ENC[AES256_GCM,data:1dSh,iv:rr9/qGYRH+BGmI+h12CmcG8qccbXhji7RPXMg3O2oTk=,tag:XeeEFc4MvUq80c9CWCL8pg==,type:str] +OTEL_SERVICE_NAME=ENC[AES256_GCM,data:geghunAlbZHbEr97CPbirtSo,iv:9DDQOV0iuBVAlLrkEMoPFAFAqxZQx+3wdea8QkzhbxM=,tag:WbpHuAka03EJLZSf82pjGg==,type:str] +HONEYCOMB_API_KEY=ENC[AES256_GCM,data:L2rw,iv:7ZXT6tLTPX8K8cdFFmr8XB9aM5PiXoLabzk+dXDAerM=,tag:RRicQ25cXLFSfXdHO0iXmw==,type:str] +API_KEY=ENC[AES256_GCM,data:qcnV,iv:8aVJLcGajFM/1Wclg6UiitCRRl94es2VBZycM0/yvII=,tag:VZVCW6CPEJeZV0T5LSfP+w==,type:str] +sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1cXRndVVCQmF5OUg3RUtW\nS0RMUGdhcFRUR0poUis1cm5nclRZbDZsVGxrCk1EVG9kRHRPRzl3ci9KcVBzQ09G\nZmFvTG5CbXA1bk5ZWHdEZGRMWk9pR0kKLS0tIFo4dytQQzlQK3lVdTdxVlZBQ2M3\nMDZYSFJZTUF4MmNtOEtQSGF4SDduM1EKevzGXoKoCU0hEWKcecBnZ6+9B3kY35c8\na9gU3yrDoMP2fA7+6Iu5+cgJt2Ise+ddGSY52kg7XobY4DCg/Ufo/A==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_0__map_recipient=age1wravpjmed26772xfjhawmnsnc4933htapg6y5xseqml0jdv8z9hqemzhcr +sops_age__list_1__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKYThXaFhnN0tVUU9sVjJ6\nRFY1Mk9rVDRFcjZ5eTZjbTNuNWpROFpXdXlRCkRNUkhhYVRjZjhRSEIxU1NJdlpD\nQVR5MDRMc2pZMnovU1NiOTdDLzBOSXMKLS0tIDhQUU1ReDlpWThPU1V2QSszd3ZT\nVHFGYW9YOTNmTmxOQlU5OGM2clIxdW8KEHDxeC7oWHY04chaVsUzutowlbQAlR14\n/sgSspLISHdXNw/7vksI62GB5OmbCF9/316LY01yClFpKvbGcZe3fA==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_1__map_recipient=age1vhftscteyrwphx0jpp0yl60xxrjs77jq05mkzv88js7ckc926vcqepp2cj +sops_age__list_2__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTdFMrWVlYWnQyS0FaR21t\na3Nsb1FSVVRxMDN5b0F6TDBoMHNDNW0yc2lFCmpKRE0yWEhPM1BWbXBJT0U1SHNl\nNzJKSnJRREZsajBCVjFreDNWcE5XNkEKLS0tIE9LMUxOdUlNQzdoNnFKRW82ZHA5\nZ3JjdHFaVG94TlRZVUpFK1BTRXVHeXMKbUrmQBwDMgY3c3OlIeN7UHQZHkHTBzBP\n16trH9mVU3vsbYCxanrY+hy6E90LeaU5tnntHXb0HMqGDl6vgCNHYQ==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_2__map_recipient=age1slx6e48k7fre0ddyu7dtm2wwcqaywn5ke2mkngym3rpazxwvvuyq9qjknk +sops_age__list_3__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5bUFyTGwwL0ZwaVArSnZ1\ndUJkQnV2NVhTQnQyK2JGSUc3aHAyZ3N5UVNVCkF5WnFPU0RxMUFqOUtQeitQbzdD\nNHRNaGZBNUtTMlJFbzcvRGo3YklmTFUKLS0tIFlQZHdTSVd1V0JERDZEcjR3Q0RZ\nZlc4c0VLalFQK2krQjcxT0ZHb21PcEkK0tfMklh/5YsvkJ4P/iIT1TjWEcmtYJob\n7kOqk4B10aA1v4NmeqGBpf+stqXebUblpQmEXq5gmMQFlawFWHFPIQ==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_3__map_recipient=age1vfhyk6wmt993dezz5wjf6n3ynkd6xptv2dr0qdl6kmttev9lh5dsfjfs3h +sops_age__list_4__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0Y0RNMGh4eTdNWWNobWlY\ncGsxTDVrZ2xmOGhnRjlob01sQ21mSmJvdWdVCmcwaVJBNnczQ2VXZmM3ZXlGbUVH\ncUtBMnRYZHk3M0xZcVZyQ29jMTVxSTAKLS0tIHdVNGNZWkJxaGZnNDBKMTFFSDJl\nbnVYZ0d1L05qbFhZWXoxazFvQ0tRYkkKA6qOZA6z/d01xmV6zomUA8JUrTUywKro\nUz0bkgsHnvElWv13CBaYEDMHZmkhWq+shnr+tYUXrXx4RLbq3a70EA==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_4__map_recipient=age1jwuvzyghfer7rx3qtqa4vs0gxyff0sg6pqgmqvp5zlhnpmvrkdlsdha4kx +sops_age__list_5__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBScWxBREtCWWZ3TXYrR3Qw\nbFA4Q05VVktiVUZ1RjZqUi9pRllwRDU3Sm1FCmJhbUZQTFZZL2kyN0tNUlQwWVlD\ndDBtekZndlgzUDlxN01MWE13ZVJaTGsKLS0tIFRpRlN4a3lOSWJuU2p3UWRsb2ti\ncnoweWV3ZlBhOEx5cXZzNTBKbDN3eTgKgV/v4wrEnBXdDe0wiT9iX/abq1RxJ6Kv\nh6USU/cUJqElUU5Nu8rYqlVyrJw9zF5hq+L0XomOGWyYdfwWW9qXaQ==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_5__map_recipient=age1pgxk292zq30wafwg03gge7hu5dlu3h7yfldp2y8kqekfaljjky7s752uwy +sops_age__list_6__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBzTS9mK0ljd1NRbUJtU2J2\nY1NHbWZzem9NZ3R6VnQvMEJnSm0zdG4xaUVZCi9zL3FNRytMUmd2eVU5ekxYZVpz\ndXVwd3Nrd25kamRMZW1wbHM0RUo2N1EKLS0tIHNSUGVESHBjRmQ0eVY4WndsRWlQ\ncWNqUUF5cUtDUFhTVFBIdTRUNjcxWDAKWtdnGqv+POmO3gyPfn4gxEF6bndbVMPN\nkr2jYlNlk+yLxin5t2eZXnwEdHKD6/JPrzobaV722Y3gcy4SMBrx3g==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_6__map_recipient=age15fsv503cl9ajfg9fphw45sfq0ytlptnr8wtlt48asytz4ev6rfks2pwt5s +sops_lastmodified=2026-04-06T10:33:35Z +sops_mac=ENC[AES256_GCM,data:FhzbTx/c9Cikj+q3nFeunA5flcDL/IDuLzbTNqFiMloqUl2nLgfqP/MnPzti3/IS45EgIQlK8Tc3prSN5B6NDKH0UN0M1xvbe9p2eeoDQJk8I/OqpqrhqkfnClbk82DIQvjps4TqhJZidu7LHV0g+lF33Uwl0F3a1rMwkRL6umo=,iv:SkKnb7RqNRQGix2rKrv0oALUDOc0leChmb+abgcB4UY=,tag:HlDtB4atTYGXEt+0OCnSMA==,type:str] +sops_unencrypted_suffix=_unencrypted +sops_version=3.12.1 diff --git a/infra/.env.sandbox.enc b/infra/.env.sandbox.enc new file mode 100644 index 0000000000..71249e26d9 --- /dev/null +++ b/infra/.env.sandbox.enc @@ -0,0 +1,53 @@ +#ENC[AES256_GCM,data:EUk6PIzXsseMFS4wuOnFQj91hGFwV1hzfi3w2DQtroWV,iv:Gr9+RpcxLEZ0/s37D5tEdejqHCKFe2PBonqh7J4NKUQ=,tag:ELZGUZkCUoirFfUC+SWTmg==,type:comment] +#ENC[AES256_GCM,data:zD2eJw0olC7BZUYmFfI+ui2rJOHotGFvfXsJtVV6Fs9eddGy2puY27q1MqfFJZ1HS8qd1YpWTqprPb0=,iv:WM22N8dLhJSE/zNA+ASjwENeSuZ9Cfx+FuF7T2tM9Yw=,tag:CBzIFOJIyHUJprY0+fAvJw==,type:comment] +#ENC[AES256_GCM,data:GgsiNd0FA23Ml4AOVn0gpzzjrFimTi6QEBIDJoWjR8lFjNSEQIbBUzdwSBUJtRW9/kpE9nciaRioyRhuzbf5riEy4g==,iv:v/rX0FvZBCVQjH6KJvOsRvBOzzapSpJ1S5xtD5BeuJ0=,tag:va2QU1onE+vV7leJjjuajA==,type:comment] +PUBPUB_HOSTNAME=ENC[AES256_GCM,data:0rThpmSzlemAo9bEedewZKEhZw==,iv:mHZIqB+V3LY7qkxq8aSmd+J8OLTpDJQAIK/r7Sr9zJU=,tag:RPtpmXfP0dzNn2h4WDREKw==,type:str] +PUBPUB_URL=ENC[AES256_GCM,data:hZlG9x0gFG0f0FzC+SPCWnV1s/Tulm18OhOY,iv:7UM8dBIV3yaGOakYSkm49dpEEnoWYojpoNppLHfPGFY=,tag:1tu98rRDaRdiD/aHBVThNw==,type:str] +POSTGRES_USER=ENC[AES256_GCM,data:uZ5vn1rC3WA=,iv:TkJZImdU6aafkJ0Ye6cdaT3/Z+w4/7QemlAS6guKCus=,tag:UjqhwiubHrWGKKZqwLvu8A==,type:str] +POSTGRES_PASSWORD=ENC[AES256_GCM,data:hCmhLFmyazk=,iv:7Cyq18Hb4iK5PQfBqb4xwJ8vL2POEJyElYUa5hx93gU=,tag:pH0L4Z1ijcO9kPMQIdKRMw==,type:str] +POSTGRES_DB=ENC[AES256_GCM,data:ISSMJidN,iv:0xkYpUb1dg9Bp13XCdIPIJwelfZGd07hNDY7pIBKJjg=,tag:wtZVSRcSuW/oSGqqHWX+4g==,type:str] +DATABASE_URL=ENC[AES256_GCM,data:3LAclVCQWrG+7CdFXPnj47QVzEvXAyUbQDesSeCl4jZ4TG6dl99x56fJTBTF+jYOszdkjnps37yQ2du6U8zFoVHUdbgh3bEJBw==,iv:sgxvIo8HqTZDrFQGN0TawKvcwnbRCV1mKf6fAwbxepU=,tag:zKeU/TY3o8vo7kWA7x05jQ==,type:str] +PGHOST=ENC[AES256_GCM,data:Wu8=,iv:bf6higOwpwLgZ2gSHU3vtsBWIknD8IqSRE+FJymZNvY=,tag:xWgR/KULnc5q+ht5r/q6sg==,type:str] +PGPORT=ENC[AES256_GCM,data:wUDbkA==,iv:vu5OWULm4wUsdvla8ng9MYYuavj28z31xRMH0FkpB5o=,tag:iie9hQz66wEC1ButsZNC0w==,type:str] +PGUSER=ENC[AES256_GCM,data:w2SIl1KPFkKE8XWEZEPApg==,iv:qcPcosmKNdNBap2ZvXkODc1+AL2LRCpuyoySXvZx97I=,tag:9l9dP2RqRXe323VIuwHbNQ==,type:str] +PGPASSWORD=ENC[AES256_GCM,data:tskXQm/SiPq4v/mtnzfvZZjqhqU=,iv:kMCrdDbO3IIYrr6Fq3wlBDnFAKeFDX0XXA5CMFfpksc=,tag:acZGDyb30y7ZShxmtIOEYw==,type:str] +PGDATABASE=ENC[AES256_GCM,data:k7pH4TISVtrgCQZBNKQ=,iv:FT/tVKCe8Uzj6fKt3tjGEQ5sm/MJZTb2cfCcbhH8yzI=,tag:XERwNJlVJya6OJyGtws6IQ==,type:str] +VALKEY_HOST=ENC[AES256_GCM,data:eOykTCQ=,iv:bEYSu68W3pHnDPJBG4qeBcnr30gWHgKRTS0QiLH6Jjk=,tag:fXp6f3nNtKnXACcRbzax5w==,type:str] +MINIO_ROOT_USER=ENC[AES256_GCM,data:B9teVQllUozOjWAclQ==,iv:WlwU2agbABSFfZ0Ai8MAf3nLmDm2f0ngXjyYV2ctZVg=,tag:pfJesr2Wc+1JkT6cS4Qdig==,type:str] +MINIO_ROOT_PASSWORD=ENC[AES256_GCM,data:m5IhnP4bmLwvhRH1MQ==,iv:YwrFzZy4V9DTgIHB6kjuaHkf9bYR3qlWOg387lioC2E=,tag:j94mHdsseLMCz2qWX1XN0w==,type:str] +ASSETS_BUCKET_NAME=ENC[AES256_GCM,data:J7+yZke6,iv:nmGO4MnTHkF02hT+Swus0AdzMUnLQAqnlVQbmaV8Ibc=,tag:+lygXbeROCrFv7WY0FniOQ==,type:str] +ASSETS_UPLOAD_KEY=ENC[AES256_GCM,data:2MvQ0tTCJKlELzw=,iv:mVOi3FAknw098g2AY+h43mbtZ9YT3PNt6fkOmzBTaKo=,tag:cDIcHbJvB8h6P5ZXULv1pA==,type:str] +ASSETS_UPLOAD_SECRET_KEY=ENC[AES256_GCM,data:fR04nwkgZ8jFC0s=,iv:lJcMu0xS5AsKN9X+LZ70RELGF4u1eI0uKTbpyQcoTQQ=,tag:SSaKB4Sn620gwXm9ilRYkQ==,type:str] +ASSETS_REGION=ENC[AES256_GCM,data:AHtDF/GoJc/C,iv:F7XuISy3MXp77A9xAaMryKs4mQZPrb7Oebl9bBVAfJQ=,tag:kK4IHFGO7tCsV+16Xg4Dng==,type:str] +ASSETS_STORAGE_ENDPOINT=ENC[AES256_GCM,data:NKCglvQc9KIoARg9kNhx8qM=,iv:d4RdlJQURPKGeNDlXDONDkTW0vA1yYCvWQZlRLQh2EI=,tag:AdpBoDVZ2/IznMTlqPIxrw==,type:str] +ASSETS_PUBLIC_ENDPOINT=ENC[AES256_GCM,data:PVQHF4MFGv02V5c5l2X/UYsix+q5E4Phim3j7qhuXkv+Nw==,iv:+wE5+eDgFT8CPqDrzXuR8Jc8sm7Et1KIGXMuMBkXK/U=,tag:hSEpd5211TbOINqpT9lpiw==,type:str] +S3_ENDPOINT=ENC[AES256_GCM,data:GE+jlsFsEhlT/NrhRASHOUo=,iv:UgnJ9aS4Et+otQCIyAQN0A6FasbHLBVxeYuG+/BW7Vc=,tag:KDtGbGOWtME8RYGMYEeY9Q==,type:str] +S3_REGION=ENC[AES256_GCM,data:bfsZhVI6SUQt,iv:eqbKTZ/JEi8CViY4nbOLIp2txcdUxXk+BFfPQt0oYXE=,tag:XpbvEPh1dPTC3dfn7HZxuA==,type:str] +SITE_BUILDER_ENDPOINT=ENC[AES256_GCM,data:WaJmPKLU0xgz2filfafc7YfCXKNpQLrt,iv:qbnYSvd+j/vJge3jdckmjhg6GKc2+QTK6EBAE34HA4g=,tag:tfbitWsNF2k6hSfLoawytQ==,type:str] +SITE_BUILDER_API_KEY=ENC[AES256_GCM,data:nUfu,iv:YkcgqQqaw3EhllxJrODxKcxK6XLRamKl6R409oP8wLs=,tag:kAei09a2qd9iAKi9AXjbXQ==,type:str] +MAILGUN_SMTP_HOST=ENC[AES256_GCM,data:jTdA,iv:ulzrjrcYakwL0CVZLmyhXkHXpq+xqXbQiQAKQL11P6s=,tag:ObCFVmmbd3VN620xGQKvxw==,type:str] +MAILGUN_SMTP_PORT=ENC[AES256_GCM,data:+PaC,iv:4yAKdeIlgnF6xHNOT0Z6CR7Nu3rG6waJCvkgfhNOFL8=,tag:jEiAaXp0kUn3s+cP8bjKyg==,type:str] +MAILGUN_SMTP_PASSWORD=ENC[AES256_GCM,data:Fkgy,iv:tDcWCgTpMQvAhMXzu/HokUCYNlzfiEc+FjGcI3VkaZQ=,tag:/a9h4511XDbmk16s+DmcCA==,type:str] +MAILGUN_SMTP_USERNAME=ENC[AES256_GCM,data:OKMx,iv:38RLXgWXNzA/XgJ22Qil4ZBo2Obo2uKHA3SYtWfOCC8=,tag:2LhnzeXIkabpXBwWXNTe9w==,type:str] +GCLOUD_KEY_FILE=ENC[AES256_GCM,data:eXcO,iv:jOd0V1n+g/CV04x92oLLD7BHj0FOaTKjTqeOv4GdT+M=,tag:M1pg4Sf/VhDIw/8dQRWtPA==,type:str] +OTEL_SERVICE_NAME=ENC[AES256_GCM,data:/osaHe00IZD9Uz/gDyca4Sg=,iv:cGZ4j0VC4v5Th8Qn9r4hT3vH6Ne6SE4EA/yLarIWHNI=,tag:an3A9I/ze6AYxwLVEJiMCA==,type:str] +HONEYCOMB_API_KEY=ENC[AES256_GCM,data:47eY,iv:tMJArSXqZiDqWiotXFmFyW0WqR+3kvkxuCk2lp2IhyU=,tag:3qWyEBnHXFlm8qfniQJ93g==,type:str] +API_KEY=ENC[AES256_GCM,data:r9MF,iv:7JBUNs84+/+l9vWP/eMY51YcvxeiCAWYfBRfKZQUSpc=,tag:i2eKY5CYm2dDJmZFV5S9Ew==,type:str] +sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBXUmtQMy9ZREpPa0NLRGsv\naU1VMW83cUJlWjFTMjJreW42UnltWUI0L2h3CmJxdzRXa2FPaERHR0xyUHY5SDFv\nb282VktXSWlrdk1NN1hSbzgzSDNhYUEKLS0tIHg0MHR5WHhJcDFna0JDbUR3c0I5\ncVJqaGw1UE9VclhOSDNaeUR0UzZVeEkKmrjBiCf23E1uxNtCXr8zpVvP2gMAdUFC\nV7fk+zpa3443cc/2AvSAiWxyXHblykN+cxc+suNUDNv1QtQlqG7RIg==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_0__map_recipient=age1wravpjmed26772xfjhawmnsnc4933htapg6y5xseqml0jdv8z9hqemzhcr +sops_age__list_1__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnME9uSXF3TU90VFFEb0Vr\nTHlFT1JCZGxUMVBLNDJ0bkdMTmU0YktNY0NRCjIvakQrRjc4OVB3WDdwWEt0Sjd0\nWUxvT001SnVsZGhrWjV1MzJjOGNnS3MKLS0tIC8xK01yYnRiMzIyTDd1dVpGaUNh\nU1lmZnREUktRRkpQOHlpcVBOQlVlVVEKQW699/iBFbah1OOVbcClLoyWadsDGYaf\nMz+5EvDFwO7LPxld3uTkXNh0tNuXmi6T85c09FM7BPqarNMsBbqVJA==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_1__map_recipient=age1vhftscteyrwphx0jpp0yl60xxrjs77jq05mkzv88js7ckc926vcqepp2cj +sops_age__list_2__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSByY3F3ZFU5KzBVYklIbmR1\nQjloZFFnZEMvekxrTUpOb3FMUnpFekZXODNZCmhRNjZiNGZjVHVlTTFPSDJnNGFG\nS3NyWFF0UFBxL0ZGYUhxeGpGVldBVEEKLS0tIGVwclczRERPTENWMGl2VmtTaTBQ\nTWVIanRTMURaWmFJSWZwTkZSRUlRbEkKqMEEWUGjLYbDwd3q8isW/mg5c+MNPcod\naSh7XqVcOyDBUBr1yuQ1DY2itMY+x502HNzUl84i67mep0k619jgTA==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_2__map_recipient=age1slx6e48k7fre0ddyu7dtm2wwcqaywn5ke2mkngym3rpazxwvvuyq9qjknk +sops_age__list_3__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBtU05jcmxMeUJWZnpISkd6\nTFZFNkxPMjBXOVNSSmxwWUNZczdtZXYxWTFVCitCclkyV2hsUnJzcGhvbXluc3N2\nOGVsUldjaEpxMVFMRVZzNm9mV0N6SjQKLS0tIHBEOFhlbDNFTThOZW1HZU0veldX\nMmZiNkZBZzA4QVlyRTJRTzBSVXJPSUkKEGiTA2DpTTOUI/dxrQSYoQhiS6DM4Q/t\nOMAA3OU05t2n0SeWUoJ5yFJ2PcFWTkXnUG173e/ip5zpdL/el7dDRw==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_3__map_recipient=age1vfhyk6wmt993dezz5wjf6n3ynkd6xptv2dr0qdl6kmttev9lh5dsfjfs3h +sops_age__list_4__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1NzZpYk9RMithS0xieWZ1\nVGo0VURsOWQ3VTJjK1V3amtVR3dCQ2twZjF3Cm5IbkliWmp5dXpFSUl1YURJelZD\nb0ZDaExNc3NNQUFpK2p0UUZCdHhMNjgKLS0tIHRRck13c2crMXF1TnFha2NpMHBD\nVlovMGxyMmx4TWVYdUF3NHhQRTVNa3cKE/patQW9NnZ6LnbGT21gs6F9ggCCbveI\nff0kfaLVFaG073PGSRlS/6pI6kY10E8VWWUTjsA1tBZM3vLsZ0fq7A==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_4__map_recipient=age1jwuvzyghfer7rx3qtqa4vs0gxyff0sg6pqgmqvp5zlhnpmvrkdlsdha4kx +sops_age__list_5__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYajZNZkxacjY5L2Q4KzBu\nS25iZ082a1FSSVpwODR5S2I1U09CTE5wWEVFCjAxOEZ4STAzcDZILzBxeWVYK090\nNTBQUlpSMmlhSDhRNkUxaWFLb1p3SlkKLS0tIE1MYXVuaEhDd1ROdGdlbUZKckZv\nRWFLRWNGUWZ2TVg1Qi9hZCttSlc4TlUKR5vNkVuEqf4sBxCPFdse4XInlB8xqA0z\n3szMwE8r8fIzKg8j8chh3yb3H2aHel2WJanF7+omB8H19Fvq1+3NcQ==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_5__map_recipient=age1pgxk292zq30wafwg03gge7hu5dlu3h7yfldp2y8kqekfaljjky7s752uwy +sops_age__list_6__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSWTRCcGowV1c2blhrZjRP\nczBBaldMemM0SGI4dzRiVWdMbEkxM0VXRlhzCkNpaVFqSnZJbU5RWkIweGo0WU9L\neW5WTzN1T1djR1Z5WlVDdmZteVJ0SmcKLS0tIHZ4WUxhdzJXQzFYTG5ET1dCUi9m\nOW5IbWpESUd6K1ZZNHh1M20rNWxTTEUKTjqTQD+SCcfLdj8J9PFMgNfUkqScy3DN\ncSYOwbpH2ObVhvvmgLqjR+RQdLkpb2itf4cufKspSi5d6uA2AQfwtw==\n-----END AGE ENCRYPTED FILE-----\n +sops_age__list_6__map_recipient=age15fsv503cl9ajfg9fphw45sfq0ytlptnr8wtlt48asytz4ev6rfks2pwt5s +sops_lastmodified=2026-04-06T10:33:40Z +sops_mac=ENC[AES256_GCM,data:oZ7HHxAKttSyIEhkQ1pWPUwKD95EXNYUXKaO90T1ye4AWRpFPcBIfOkCk0eyOJyD1iPpj7MgzTcNcyCC14Q4G2ZnNQE/Gy4ma4UQSCcGqG85AqYWJ81Blj4thzWxZLpE1ZDqGNJilI0zQeMu2fH39rrFrBqt5b81X6TKYMCjvEc=,iv:sBkqMnLIEHmg8Bxv/wO109qKItTO+zFfGvHAhwwLoqI=,tag:5t82yBl7V6ITFVoh3imWoA==,type:str] +sops_unencrypted_suffix=_unencrypted +sops_version=3.12.1 diff --git a/infra/.sops.yaml b/infra/.sops.yaml new file mode 100644 index 0000000000..40eddb2aa2 --- /dev/null +++ b/infra/.sops.yaml @@ -0,0 +1,17 @@ +# creation_rules: +# - path_regex: \.env(\.staging)?(\.enc)?$ +# age: +# # add your age public keys here, one per team member / CI system +# # generate with: age-keygen +# # example: +# # - age1xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +creation_rules: + - path_regex: \.env(\.preview|\.sandbox)?(\.enc)?$ + age: + - age1wravpjmed26772xfjhawmnsnc4933htapg6y5xseqml0jdv8z9hqemzhcr + - age1vhftscteyrwphx0jpp0yl60xxrjs77jq05mkzv88js7ckc926vcqepp2cj + - age1slx6e48k7fre0ddyu7dtm2wwcqaywn5ke2mkngym3rpazxwvvuyq9qjknk + - age1vfhyk6wmt993dezz5wjf6n3ynkd6xptv2dr0qdl6kmttev9lh5dsfjfs3h + - age1jwuvzyghfer7rx3qtqa4vs0gxyff0sg6pqgmqvp5zlhnpmvrkdlsdha4kx + - age1pgxk292zq30wafwg03gge7hu5dlu3h7yfldp2y8kqekfaljjky7s752uwy + - age15fsv503cl9ajfg9fphw45sfq0ytlptnr8wtlt48asytz4ev6rfks2pwt5s diff --git a/infra/Caddyfile b/infra/Caddyfile new file mode 100644 index 0000000000..d65c80d7de --- /dev/null +++ b/infra/Caddyfile @@ -0,0 +1,40 @@ +{ + admin :2019 +} + +{$PUBPUB_HOSTNAME} { + encode gzip + + handle_path /assets* { + reverse_proxy minio:9000 + } + + handle_path /assets-ui* { + reverse_proxy minio:9001 + } + + handle_path /site-builder* { + reverse_proxy site-builder:4000 + } + + handle_path /sites/* { + root * /sites + file_server { + fs s3 { + bucket {$S3_BUCKET_NAME:assets} + region {$S3_REGION:us-east-1} + endpoint {$S3_ENDPOINT:http://minio:9000} + access_key {$S3_ACCESS_KEY} + secret_key {$S3_SECRET_KEY} + } + } + } + + handle { + reverse_proxy platform:3000 + } +} + +:80 { + respond "OK" 200 +} diff --git a/infra/Caddyfile.preview b/infra/Caddyfile.preview new file mode 100644 index 0000000000..722a4fe3a7 --- /dev/null +++ b/infra/Caddyfile.preview @@ -0,0 +1,81 @@ +{ + admin :2019 + debug + on_demand_tls { + ask http://localhost:8888/allow-domain + } +} + +# this only allows preview-*.pubstar.org domains to be used for TLS certificates +# prevents abuse +:8888 { + @allow expression {path} == "/allow-domain" && {query.domain}.matches("pr-[0-9]+.pubstar.org") + handle @allow { + respond "OK" 200 + } + + respond "Not allowed" 403 +} + + +(s3site) { + # strip trailing slash from paths (except root) + @pathWithSlash path_regexp dir (.+)/$ + handle @pathWithSlash { + redir {re.dir.1} permanent + } + + # rewrite to include bucket path + rewrite * /{args[0]}{uri} + + # reverse proxy to garage S3 API + reverse_proxy {args[1]} { + # handle 403/404 by trying index.html + @error status 403 404 + handle_response @error { + rewrite * {uri}/index.html + reverse_proxy {args[1]} { + @nestedError status 404 + handle_response @nestedError { + respond "Not found" 404 + } + } + } + } +} + +:443 { + tls internal { + on_demand + } + + encode gzip + + handle_path /assets* { + reverse_proxy minio:9000 + } + + handle_path /assets-ui* { + reverse_proxy minio:9001 + } + + handle_path /site-builder* { + reverse_proxy site-builder:4000 + } + + handle_path /sites/* { + import s3site {$S3_BUCKET_NAME:assets} {$S3_ENDPOINT:http://minio:9000} + } + + handle_path /emails/* { + reverse_proxy inbucket:9000 + } + + handle { + reverse_proxy platform:3000 + } +} + +:80 { + respond "OK" 200 +} diff --git a/infra/import-backup.sh b/infra/import-backup.sh new file mode 100755 index 0000000000..293453ad43 --- /dev/null +++ b/infra/import-backup.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +DUMP_FILE="${1:?usage: $0 }" +STACK_NAME="${2:-pubpub}" + +if [[ ! -f "$DUMP_FILE" ]]; then + echo "file not found: $DUMP_FILE" + exit 1 +fi + +DB_CONTAINER=$(sudo docker ps --filter "label=com.docker.swarm.service.name=${STACK_NAME}_db" --format '{{.ID}}' | head -1) + +if [[ -z "$DB_CONTAINER" ]]; then + echo "no running db container found for stack: $STACK_NAME" + exit 1 +fi + +echo "importing $DUMP_FILE into container $DB_CONTAINER ..." + +if [[ "$DUMP_FILE" == *.sql ]]; then + sudo docker exec -i "$DB_CONTAINER" \ + psql -U "$PGUSER" -d "$PGDATABASE" < "$DUMP_FILE" +else + sudo docker exec -i "$DB_CONTAINER" \ + pg_restore --clean --if-exists --no-owner -U "$PGUSER" -d "$PGDATABASE" < "$DUMP_FILE" +fi + +echo "import complete" diff --git a/infra/stack.preview.yml b/infra/stack.preview.yml new file mode 100644 index 0000000000..ef1560e4c6 --- /dev/null +++ b/infra/stack.preview.yml @@ -0,0 +1,147 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/swarmlibs/dockerstack-schema/main/schema/dockerstack-spec.json + +services: + + proxy: + image: caddy:latest + env_file: [.env] + volumes: + - ./Caddyfile.preview:/etc/caddy/Caddyfile:ro + - caddy_data:/data + - caddy_config:/config + networks: [appnet] + ports: + - target: 80 + published: 80 + protocol: tcp + mode: host + - target: 443 + published: 443 + protocol: tcp + mode: host + deploy: + replicas: 1 + restart_policy: + condition: any + + platform: + image: ghcr.io/knowledgefutures/platform:${IMAGE_TAG} + env_file: [.env] + environment: + HOSTNAME: "0.0.0.0" + NODE_ENV: production + PORT: "3000" + PUBPUB_URL: https://pr-${PR_NUMBER}.pubstar.org + PUBPUB_HOSTNAME: pr-${PR_NUMBER}.pubstar.org + SITE_BUILDER_ENDPOINT: http://site-builder:4000 + S3_PUBLIC_ENDPOINT: https://pr-${PR_NUMBER}.pubstar.org/assets + FLAGS: "uploads:off,invites:off,disabled-actions:http+email" + DB_RESET: "true" + DB_SEED: "true" + networks: [appnet] + healthcheck: + test: + - CMD-SHELL + - > + node -e "require('http') + .get('http://127.0.0.1:3000/api/health', r => process.exit(r.statusCode < 400 ? 0 : 1)) + .on('error', () => process.exit(1));" + interval: 10s + timeout: 3s + retries: 6 + start_period: 60s + deploy: + replicas: 1 + restart_policy: + condition: on-failure + + platform-jobs: + image: ghcr.io/knowledgefutures/platform-jobs:${IMAGE_TAG} + env_file: [.env] + environment: + NODE_ENV: production + PUBPUB_URL: http://platform:3000 + networks: [appnet] + deploy: + replicas: 1 + restart_policy: + condition: on-failure + + site-builder: + image: ghcr.io/knowledgefutures/platform-site-builder:${IMAGE_TAG} + env_file: [.env] + environment: + NODE_ENV: production + PUBPUB_URL: http://platform:3000 + PORT: "4000" + networks: [appnet] + deploy: + replicas: 1 + restart_policy: + condition: on-failure + + db: + image: postgres:15 + env_file: [.env] + volumes: + - pgdata:/var/lib/postgresql/data + networks: [appnet] + deploy: + replicas: 1 + restart_policy: + condition: any + + cache: + image: valkey/valkey:8-alpine + networks: [appnet] + deploy: + replicas: 1 + restart_policy: + condition: any + + minio: + image: minio/minio:latest + env_file: [.env] + command: server --console-address ":9001" /data + environment: + MINIO_BROWSER_REDIRECT_URL: https://pr-${PR_NUMBER}.pubstar.org/assets-ui + networks: [appnet] + deploy: + replicas: 1 + restart_policy: + condition: any + + minio-init: + image: minio/mc:latest + env_file: [.env] + entrypoint: > + /bin/sh -c ' + /usr/bin/mc alias set myminio http://minio:9000 "$${MINIO_ROOT_USER}" "$${MINIO_ROOT_PASSWORD}"; + /usr/bin/mc mb --ignore-existing myminio/"$${S3_BUCKET_NAME}"; + /usr/bin/mc anonymous set download myminio/"$${S3_BUCKET_NAME}"; + /usr/bin/mc admin user add myminio "$${S3_ACCESS_KEY}" "$${S3_SECRET_KEY}"; + /usr/bin/mc admin policy attach myminio readwrite --user "$${S3_ACCESS_KEY}";' + networks: [appnet] + deploy: + mode: replicated-job + replicas: 1 + restart_policy: + condition: on-failure + max_attempts: 5 + + inbucket: + image: inbucket/inbucket:latest + networks: [appnet] + deploy: + replicas: 1 + restart_policy: + condition: any + +networks: + appnet: + driver: overlay + +volumes: + pgdata: + caddy_data: + caddy_config: diff --git a/infra/stack.yml b/infra/stack.yml new file mode 100644 index 0000000000..faf65a671a --- /dev/null +++ b/infra/stack.yml @@ -0,0 +1,142 @@ +services: + proxy: + image: ghcr.io/knowledgefutures/caddy-sites:latest + env_file: [.env] + ports: + - target: 80 + published: 80 + protocol: tcp + mode: host + - target: 443 + published: 443 + protocol: tcp + mode: host + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + - caddy_data:/data + - caddy_config:/config + networks: [appnet] + deploy: + replicas: 1 + restart_policy: + condition: any + + platform: + image: ghcr.io/knowledgefutures/platform:${IMAGE_TAG} + env_file: [.env] + environment: + HOSTNAME: '0.0.0.0' + NODE_ENV: production + PORT: '3000' + PUBPUB_URL: ${PUBPUB_URL} + SITE_BUILDER_ENDPOINT: http://site-builder:4000 + networks: [appnet] + healthcheck: + test: + - CMD-SHELL + - > + node -e "require('http') + .get('http://127.0.0.1:3000/api/health', r => process.exit(r.statusCode < 400 ? 0 : 1)) + .on('error', () => process.exit(1));" + interval: 10s + timeout: 3s + retries: 6 + start_period: 60s + deploy: + replicas: 2 + update_config: + order: start-first + parallelism: 1 + delay: 5s + failure_action: rollback + rollback_config: + order: stop-first + parallelism: 1 + restart_policy: + condition: on-failure + + platform-jobs: + image: ghcr.io/knowledgefutures/platform-jobs:${IMAGE_TAG} + env_file: [.env] + environment: + NODE_ENV: production + PUBPUB_URL: http://platform:3000 + networks: [appnet] + deploy: + replicas: 1 + update_config: + order: start-first + parallelism: 1 + delay: 5s + failure_action: rollback + restart_policy: + condition: on-failure + + site-builder: + image: ghcr.io/knowledgefutures/platform-site-builder:${IMAGE_TAG} + env_file: [.env] + environment: + NODE_ENV: production + PUBPUB_URL: http://platform:3000 + PORT: '4000' + networks: [appnet] + deploy: + replicas: 1 + update_config: + order: start-first + parallelism: 1 + delay: 5s + failure_action: rollback + restart_policy: + condition: on-failure + + db: + image: postgres:15 + tmpfs: + - /dev/shm:size=2147483648 + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + command: > + -c shared_buffers=2GB + -c effective_cache_size=6GB + -c work_mem=16MB + -c maintenance_work_mem=512MB + -c max_connections=500 + volumes: + - pgdata:/var/lib/postgresql/data + networks: [appnet, dbaccess] + deploy: + replicas: 1 + restart_policy: + condition: any + + cache: + image: valkey/valkey:8-alpine + networks: [appnet] + deploy: + replicas: 1 + restart_policy: + condition: any + + inbucket: + image: inbucket/inbucket:latest + networks: [appnet] + deploy: + replicas: 1 + restart_policy: + condition: any + +networks: + appnet: + driver: overlay + dbaccess: + driver: overlay + attachable: true + +volumes: + pgdata: + minio_data: + caddy_data: + caddy_config: diff --git a/package.json b/package.json index 58a7c2e858..84f24cfdaa 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,13 @@ "dev:teardown": "docker compose -f docker-compose.dev.yml down -v", "integration:setup": "docker compose -f docker-compose.test.yml --profile integration up -d", "integration:teardown": "docker compose -f docker-compose.test.yml --profile integration down -v", - "context-editor:playwright": "pnpm --filter context-editor run playwright:test" + "context-editor:playwright": "pnpm --filter context-editor run playwright:test", + "secrets:encrypt": "cd infra && sops -e --input-type dotenv --output-type dotenv --output .env.enc .env", + "secrets:encrypt:preview": "cd infra && sops -e --input-type dotenv --output-type dotenv --output .env.preview.enc .env.preview", + "secrets:encrypt:sandbox": "cd infra && sops -e --input-type dotenv --output-type dotenv --output .env.sandbox.enc .env.sandbox", + "secrets:decrypt": "cd infra && sops -d --input-type dotenv --output-type dotenv --output .env .env.enc", + "secrets:decrypt:preview": "cd infra && sops -d --input-type dotenv --output-type dotenv --output .env.preview .env.preview.enc", + "secrets:decrypt:sandbox": "cd infra && sops -d --input-type dotenv --output-type dotenv --output .env.sandbox .env.sandbox.enc" }, "devDependencies": { "@babel/core": "7.28.3", diff --git a/packages/context-editor/src/stories/mockUtils.ts b/packages/context-editor/src/stories/mockUtils.ts index c4415b9f73..c966cebd02 100644 --- a/packages/context-editor/src/stories/mockUtils.ts +++ b/packages/context-editor/src/stories/mockUtils.ts @@ -20,18 +20,18 @@ export const getPubs = async (filter: string) => { * Requires minio server to be running * */ const getS3Client = () => { - const region = import.meta.env.STORYBOOK_ASSETS_REGION - const key = import.meta.env.STORYBOOK_ASSETS_UPLOAD_KEY - const secret = import.meta.env.STORYBOOK_ASSETS_UPLOAD_SECRET_KEY + const region = import.meta.env.STORYBOOK_S3_REGION + const key = import.meta.env.STORYBOOK_S3_ACCESS_KEY + const secret = import.meta.env.STORYBOOK_S3_SECRET_KEY const s3Client = new S3Client({ - endpoint: import.meta.env.STORYBOOK_ASSETS_STORAGE_ENDPOINT, + endpoint: import.meta.env.STORYBOOK_S3_ENDPOINT, region: region, credentials: { accessKeyId: key, secretAccessKey: secret, }, - forcePathStyle: import.meta.env.STORYBOOK_ASSETS_STORAGE_ENDPOINT, // Required for MinIO + forcePathStyle: import.meta.env.STORYBOOK_S3_ENDPOINT, // Required for MinIO }) return s3Client @@ -40,7 +40,7 @@ const getS3Client = () => { export const generateSignedAssetUploadUrl = async (key: string) => { const client = getS3Client() - const bucket = import.meta.env.STORYBOOK_ASSETS_BUCKET_NAME + const bucket = import.meta.env.STORYBOOK_S3_BUCKET_NAME const command = new PutObjectCommand({ Bucket: bucket, Key: key, diff --git a/self-host/.env.example b/self-host/.env.example index e8faa80fd0..fc212917c0 100644 --- a/self-host/.env.example +++ b/self-host/.env.example @@ -17,15 +17,15 @@ POSTGRES_PORT=5432 # don't forget to update the port in docker-compose.yml if yo MINIO_ROOT_USER= # change this! this is the username for your file server! MINIO_ROOT_PASSWORD= # change this! this is the password for your file server! -ASSETS_BUCKET_NAME=assets -ASSETS_UPLOAD_KEY= # change this! example: asset-user -ASSETS_UPLOAD_SECRET_KEY= # change this! -ASSETS_REGION=us-east-1 # leave this unchanged, unless you are hosting files on a different region on actual AWS +S3_BUCKET_NAME=assets +S3_ACCESS_KEY= # change this! example: asset-user +S3_SECRET_KEY= # change this! +S3_REGION=us-east-1 # leave this unchanged, unless you are hosting files on a different region on actual AWS # this is the default value but you ideally should set this up more nicely using our caddy service -ASSETS_STORAGE_ENDPOINT="http://localhost:9000" +S3_ENDPOINT="http://localhost:9000" # you could also set this to the secured endpoint of your file server -# ASSETS_STORAGE_ENDPOINT="https://example.com/assets" +# S3_ENDPOINT="https://example.com/assets" MAILGUN_SMTP_HOST=localhost MAILGUN_SMTP_PORT=54325 diff --git a/self-host/README.md b/self-host/README.md index 60537c378a..25e1fa9871 100644 --- a/self-host/README.md +++ b/self-host/README.md @@ -92,11 +92,11 @@ The hosted version of Platfrom uses AWS S3 to host files. When self-hosting, you If you want to use your own S3-compatible storage service, you will need to set the following environment variables: ```sh -ASSETS_BUCKET_NAME="your-bucket-name" -ASSETS_UPLOAD_KEY="your-access-key" -ASSETS_UPLOAD_SECRET_KEY="your-secret-key" -ASSETS_REGION="your-region" -ASSETS_STORAGE_ENDPOINT="your-storage-endpoint" # only necessary if you are using non-AWS S3-compatible storage service +S3_BUCKET_NAME="your-bucket-name" +S3_ACCESS_KEY="your-access-key" +S3_SECRET_KEY="your-secret-key" +S3_REGION="your-region" +S3_ENDPOINT="your-storage-endpoint" # only necessary if you are using non-AWS S3-compatible storage service ``` You should also remove the `minio` and `minio-init` services from the `docker-compose.yml` file. @@ -123,7 +123,7 @@ openssl rand -base64 32 [System.Web.Security.Membership]::GeneratePassword(32,8) ``` -Run one of these commands twice, and use one for `MINIO_ROOT_PASSWORD` and one for `ASSETS_UPLOAD_SECRET_KEY`. +Run one of these commands twice, and use one for `MINIO_ROOT_PASSWORD` and one for `S3_SECRET_KEY`. ```sh # not needed if you're using a remote file server like AWS S3 @@ -131,10 +131,10 @@ MINIO_ROOT_USER= # change this! this is the username for your file server! MINIO_ROOT_PASSWORD= # change this! this is the password for your file server! # these are either the values of an existing S3-compatible storage service, or the values that will be used to create a new MinIO service -ASSETS_BUCKET_NAME= # example: assets -ASSETS_UPLOAD_KEY= # example: asset-user -ASSETS_UPLOAD_SECRET_KEY= # example: a strong secure password -ASSETS_REGION=us-east-1 # leave this unchanged, unless you are hosting files on a different region on actual AWS +S3_BUCKET_NAME= # example: assets +S3_ACCESS_KEY= # example: asset-user +S3_SECRET_KEY= # example: a strong secure password +S3_REGION=us-east-1 # leave this unchanged, unless you are hosting files on a different region on actual AWS ``` Then, after running `docker compose up -d`, you should be able to visit the MinIO console at `http://localhost:9001`. diff --git a/self-host/caddy/Caddyfile b/self-host/caddy/Caddyfile index f1389fe552..d381c02f35 100644 --- a/self-host/caddy/Caddyfile +++ b/self-host/caddy/Caddyfile @@ -23,11 +23,11 @@ example.com { root * /sites file_server { fs s3 { - bucket {$ASSETS_BUCKET_NAME:assets} + bucket {$S3_BUCKET_NAME:assets} region {$S3_REGION:us-east-1} endpoint {$S3_ENDPOINT:http://minio:9000} - access_key {$ASSETS_UPLOAD_KEY} - secret_key {$ASSETS_UPLOAD_SECRET_KEY} + access_key {$S3_ACCESS_KEY} + secret_key {$S3_SECRET_KEY} } } } @@ -40,7 +40,7 @@ example.com { # if you want to use a different domain for your files, you can do so here # for instance, now all your files will be accessible at assets.example.com -# if you go this route, be sure to update your ASSETS_STORAGE_ENDPOINT in .env and restart your services +# if you go this route, be sure to update your S3_ENDPOINT in .env and restart your services # assets.example.com { # reverse_proxy minio:9000 # } diff --git a/self-host/docker-compose.yml b/self-host/docker-compose.yml index 8d19b2f79a..29741a1611 100644 --- a/self-host/docker-compose.yml +++ b/self-host/docker-compose.yml @@ -10,8 +10,6 @@ services: condition: service_started platform-jobs: condition: service_started - platform-migrations: - condition: service_completed_successfully minio-init: condition: service_completed_successfully platform: linux/amd64 @@ -24,12 +22,12 @@ services: networks: - app-network - # platfrom jobs service + # platform jobs service # takes care of longer running tasks like scheduling actions platform-jobs: depends_on: - platform-migrations: - condition: service_completed_successfully + db: + condition: service_started platform: linux/amd64 image: ghcr.io/pubpub/platform-jobs:latest env_file: .env @@ -40,19 +38,6 @@ services: networks: - app-network - platform-migrations: - platform: linux/amd64 - depends_on: - db: - condition: service_started - image: ghcr.io/pubpub/platform-migrations:latest - env_file: .env - environment: - DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} - command: ["pnpm", "--filter", "core", "migrate-docker"] - networks: - - app-network - # cache cache: image: valkey/valkey:8-alpine diff --git a/self-host/minio-init.sh b/self-host/minio-init.sh index 42f08174ce..cb458681c7 100644 --- a/self-host/minio-init.sh +++ b/self-host/minio-init.sh @@ -1,5 +1,5 @@ /usr/bin/mc alias set myminio http://minio:9000 "${MINIO_ROOT_USER}" "${MINIO_ROOT_PASSWORD}"; -/usr/bin/mc mb --ignore-existing myminio/"${ASSETS_BUCKET_NAME}"; -/usr/bin/mc anonymous set download myminio/"${ASSETS_BUCKET_NAME}"; -/usr/bin/mc admin user add myminio "${ASSETS_UPLOAD_KEY}" "${ASSETS_UPLOAD_SECRET_KEY}"; -/usr/bin/mc admin policy attach myminio readwrite --user "${ASSETS_UPLOAD_KEY}"; \ No newline at end of file +/usr/bin/mc mb --ignore-existing myminio/"${S3_BUCKET_NAME}"; +/usr/bin/mc anonymous set download myminio/"${S3_BUCKET_NAME}"; +/usr/bin/mc admin user add myminio "${S3_ACCESS_KEY}" "${S3_SECRET_KEY}"; +/usr/bin/mc admin policy attach myminio readwrite --user "${S3_ACCESS_KEY}"; \ No newline at end of file