diff --git a/.github/workflows/update-docs-base-bun.yml b/.github/workflows/update-docs-base-bun.yml index 99c947a856..514a5144d6 100644 --- a/.github/workflows/update-docs-base-bun.yml +++ b/.github/workflows/update-docs-base-bun.yml @@ -21,6 +21,10 @@ env: # Every version listed here is pulled in and published by a single nightly build. DOC_VERSIONS: "63,62" +concurrency: + group: update-docs + cancel-in-progress: true + jobs: params: name: Prep params @@ -51,14 +55,18 @@ jobs: needs: params name: Build docs v${{ needs.params.outputs.versions }} runs-on: ubuntu-latest - # The docs pull step can retry up to 3x30 minutes on its own, so leave room for - # that plus the astro build and htmlproofer. + # The docs pull step retries and is capped at 90 minutes, so leave room for that + # plus the astro build and htmlproofer. timeout-minutes: 150 env: GH_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }} DOC_VERSIONS: ${{ needs.params.outputs.versions }} + # Every run gets its own branch, so nothing has to be merged by hand. + BRANCH_NAME: docs-update-${{ github.run_id }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 + with: + ref: master - name: Prepare java uses: actions/setup-java@v4 @@ -72,10 +80,6 @@ jobs: cli: 1.12.0.1530 bb: 1.12.200 - - name: Run docs repo build tests - run: | - bb script/_test/all.clj - - name: Filter non-documented branches run: | for v in ${DOC_VERSIONS//,/ }; do @@ -93,59 +97,36 @@ jobs: - name: Install js dependencies run: bun install --frozen-lockfile - - name: Setup Git Authentication - run: | - git config --global user.name "Metabase Docs bot" - git config --global user.email "metabase-bot@metabase.com" - git remote set-url origin "https://x-access-token:${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}@github.com/metabase/docs.metabase.github.io.git" - # observed this failing during sdk docs generation deps downloads, so adding retry logic # note: a retry re-pulls every version, since the whole loop is the retried unit - name: Update docs for branchname - uses: nick-fields/retry@v3 - with: - timeout_minutes: 30 - max_attempts: 3 - retry_on: error - shell: bash - command: | + timeout-minutes: 90 + run: | + pull_docs() { for v in ${DOC_VERSIONS//,/ }; do echo "::group::Pulling docs from release-x.$v.x" bb script/update_docs_for_branchname.clj --source-branch "release-x.$v.x" --target-branch "release-x.$v.x" echo "::endgroup::" done + } + export -f pull_docs + + for attempt in 1 2 3; do + echo "Pulling docs (attempt $attempt of 3)" + status=0 + # the subshell restores `set -e`, so a failed version aborts the attempt + bash -euo pipefail -c pull_docs || status=$? + if (( status == 0 )); then exit 0; fi + echo "::warning::Attempt $attempt failed (exit $status)" + sleep 10 + done + echo "::error::Docs pull failed after 3 attempts" + exit 1 - name: Cleanup cloud docs (move them to latest, remove from all other versions) run: | bb script/cleanup_cloud_docs.clj - - name: "bun lint-markdown" - run: | - bun run lint-markdown - - - name: "bun lint-styles" - run: | - bun run lint-styles - - - name: "bun lint-scripts" - run: | - bun run lint-scripts - - - name: "bun lint-links" - run: | - bun run lint-links - - - name: List SDK files (debugging) - run: | - cd _docs/v0.57/embedding/sdk/ - ls -lR - - - name: Upload Master doc files (debugging) - uses: actions/upload-artifact@v4 - with: - name: sdk-files-debug - path: _docs/v0.57/ - - name: Build the Docs Site env: NODE_ENV: production @@ -155,22 +136,57 @@ jobs: # Build the docs site: script/build - - name: Run htmlproofer + metabase.com-aware link checks - run: | - SECONDS=0 - script/links || true - echo "htmlproofer took: ${SECONDS} seconds." - echo 'checking reported links...' - echo "htmlproofer spit out a report of length: $(wc -l < htmlproofer.out)" - bb script/analyze_links.clj --htmlproofer-output htmlproofer.out --limit 1 - - - name: Update or Create the Pull Request - run: | - bb script/update_or_create_pr.clj \ - --versions "$DOC_VERSIONS" \ - --pr-number "$PR_NUMBER" \ - --update-dirs "$UPDATE_DIRS" + - name: Push the updated docs + id: diff + run: | # sh + git config --global user.name "Metabase Automation User" + git config --global user.email "metabase-bot@metabase.com" + # The current docs version is also published to _docs/latest + latest="$(sed -n 's/^docs_version: v0\.\([0-9]*\).*/\1/p' _config.yml)" + for v in ${DOC_VERSIONS//,/ }; do + git add "_docs/v0.$v" || echo "::warning::nothing to stage for _docs/v0.$v" + if [[ "$v" == "$latest" ]]; then + git add _docs/latest + fi + done - - name: Merge Updated Docs - run: | - bb script/merge.clj --versions "$DOC_VERSIONS" + if git diff --cached --quiet; then + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "No docs changes to commit." + exit 0 + fi + + echo "changed=true" >> "$GITHUB_OUTPUT" + git --no-pager diff --cached --stat + + git checkout -B "$BRANCH_NAME" + git commit -m "Update docs for v$DOC_VERSIONS" + git push --force origin "$BRANCH_NAME" + + - name: Open PR and enable automerge + id: create_pr + if: steps.diff.outputs.changed == 'true' + run: | # sh + { + echo "Automated docs update, pulled from:" + for v in ${DOC_VERSIONS//,/ }; do + echo "- \`release-x.$v.x\`" + done + } > "$RUNNER_TEMP/pr-body.md" + + URL=$(gh pr create \ + --base master \ + --head "$BRANCH_NAME" \ + --title "Update docs: v0.${DOC_VERSIONS//,/, v0.}" \ + --body-file "$RUNNER_TEMP/pr-body.md") + + echo "pr_number=${URL##*/}" >> "$GITHUB_OUTPUT" + gh pr merge --squash --auto "$URL" + + - name: Auto approve PR + if: steps.diff.outputs.changed == 'true' + env: + # Must be GITHUB_TOKEN, not the job's PAT: the PAT opened the PR and + # GitHub rejects self-approval. + GH_TOKEN: ${{ github.token }} + run: gh pr review "${{ steps.create_pr.outputs.pr_number }}" --approve diff --git a/.github/workflows/update-version-support.yml b/.github/workflows/update-version-support.yml index 3d933d1a9f..21e493444a 100644 --- a/.github/workflows/update-version-support.yml +++ b/.github/workflows/update-version-support.yml @@ -72,11 +72,9 @@ jobs: git commit -m "Update major version support data" git push --force origin "$BRANCH_NAME" - - name: Open PR + - name: Open PR and enable automerge id: create_pr if: steps.diff.outputs.changed == 'true' - env: - GH_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }} run: | # sh DATE=$(date +'%Y-%m-%d') @@ -86,19 +84,13 @@ jobs: --title "Update version support data $DATE" \ --body "Automated nightly update of major version support data.") - PR_NUMBER=${URL##*/} - echo "pr_number=$PR_NUMBER" >> "$GITHUB_OUTPUT" + echo "pr_number=${URL##*/}" >> "$GITHUB_OUTPUT" + gh pr merge --squash --auto "$URL" - name: Auto approve PR if: steps.diff.outputs.changed == 'true' - uses: juliangruber/approve-pull-request-action@68fcc9a5a73b5641cadf757cf99d73720dcb05d0 # v2.1.0 - with: - github-token: ${{ github.token }} - number: ${{ steps.create_pr.outputs.pr_number }} - - - name: Enable Pull Request Automerge - if: steps.diff.outputs.changed == 'true' - run: gh pr merge --squash --auto "$PR_NUMBER" env: - GH_TOKEN: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }} - PR_NUMBER: ${{ steps.create_pr.outputs.pr_number }} + # Must be GITHUB_TOKEN, not the job's PAT: the PAT opened the PR and + # GitHub rejects self-approval. + GH_TOKEN: ${{ github.token }} + run: gh pr review "${{ steps.create_pr.outputs.pr_number }}" --approve diff --git a/script/_test/all.clj b/script/_test/all.clj index 93027a1ef5..896332b424 100644 --- a/script/_test/all.clj +++ b/script/_test/all.clj @@ -63,35 +63,6 @@ (is (integer? docs-version) (str "Expected config version to be an integer, got: " docs-version)))) -(deftest parse-versions-test - (is (= [63 62] (u/parse-versions "63,62"))) - (is (= [63 62] (u/parse-versions " 62 , 63 ")) - "whitespace is trimmed and versions are sorted newest first") - (is (= [63] (u/parse-versions "63,63")) "duplicates collapse") - (is (= [63] (u/parse-versions "63"))) - (is (= [63] (u/parse-versions 63)) "babashka.cli may hand us a number for a single version") - (doseq [bad ["" "," "63,x" "v63" "63,-1" "63,0"]] - (is (thrown? clojure.lang.ExceptionInfo (u/parse-versions bad)) - (str "Expected " (pr-str bad) " to be rejected")))) - -(deftest versions->head-ref-name-test - (is (= "docs-update-v63-v62" (u/versions->head-ref-name [63 62]))) - (is (= "docs-update-v63-v62" (u/versions->head-ref-name [62 63])) - "the branch name depends on the set of versions, not the order they were listed in")) - -(deftest versions->artifacts-test - (let [current (u/config-docs-version) - previous (dec current) - artifacts (u/versions->artifacts [current previous])] - (is (= (count artifacts) (count (distinct artifacts))) - (str "Expected no duplicate paths, got: " (pr-str artifacts))) - (is (some #{"_docs/latest"} artifacts) - "the current version also publishes to _docs/latest") - (is (some #{(str "_docs/v0." current)} artifacts)) - (is (some #{(str "_docs/v0." previous)} artifacts)) - (is (not (some #{"_docs/master"} artifacts)) - "master docs are never published"))) - (deftest categorize-branchname-test (doseq [branchname branches :let [[category release-num] (u/categorize-branchname branchname)]] diff --git a/script/merge.clj b/script/merge.clj deleted file mode 100644 index 4ffa9c28d6..0000000000 --- a/script/merge.clj +++ /dev/null @@ -1,183 +0,0 @@ -(ns merge - (:require - [babashka.cli :as cli] - [babashka.fs :as fs] - [babashka.process :as p] - [cheshire.core :as json] - [clojure.string :as str] - [ice.core :as ice] - [util :as u])) - -(def cli-spec - {:spec - {:versions {:ref "" - :desc "Comma separated major versions included in this build, eg. \"63,62\"." - :alias :v - :require true}} - :error-fn u/cli-error-fn}) - -(defn- find-pr-list [head-ref] - (let [pr (-> (p/sh "gh" "pr" "list" - "--head" head-ref - "--json" "number,headRefName") - :out - (json/parse-string true) - first)] - (if pr - (do (ice/p [:green "Found PR #" (:number pr)]) - (:number pr)) - (throw (ex-info - (str "No PR found for " head-ref) - {:head-ref head-ref - :babashka/exit 1}))))) - -(defn- find-pr-view [head-ref] - (let [pr-num (-> (p/sh "gh" "pr" "view" head-ref - "--json" "number" - "--jq" ".number") - :out - str/trim)] - (when pr-num (parse-long pr-num)))) - -(defn- resolve-conflicts-for-file [file strat] - (ice/p [:yellow "Resolving file: " file]) - (:out (p/sh "git" "checkout" strat file)) - (ice/p [:yellow " - git checkout " strat " " file]) - (:out (p/sh "git" "add" file)) - (ice/p [:yellow " - git add " file])) - -(defn- resolve-conflicts - "Resolve conflicts by auto-merging changes in artifact directories based on merge-strategy. - If merge-strategy is :ours, prefer changes from the PR branch. - If merge-strategy is :theirs, prefer changes from the target branch." - [artifacts merge-strategy] - (let [conflicted-files (->> (p/shell {:out :string :continue true} - "git" "diff" "--name-only" "--diff-filter=U") - :out - str/trim - str/split-lines - (remove str/blank?)) - strat (case merge-strategy - :ours "--ours" - :theirs "--theirs")] - (if (empty? conflicted-files) - (ice/p [:green "No conflicts to resolve"]) - (do - (ice/p [:blue "Conflicted files: " (str/join ", " conflicted-files)]) - (ice/p [:blue "Artifact directories: " (str/join ", " artifacts)]) - (doseq [artifact artifacts] - (if (fs/directory? artifact) - (let [files-in-dir (filter #(str/starts-with? % artifact) conflicted-files)] - (when (seq files-in-dir) - (ice/p [:yellow "Resolving conflicts in directory: " artifact]) - (doseq [file files-in-dir] - (resolve-conflicts-for-file file strat)))) - (resolve-conflicts-for-file artifact strat))))))) - -(defn- update-and-merge-pr [head-ref-name versions pr-number merge-strategy] - (ice/p [:blue "Updating PR branch..."]) - (ice/p [:blue "Attempting merge with origin/master..."]) - (let [merge-result (p/sh {:continue true} "git" "merge" "origin/master")] - (when-not (zero? (:exit merge-result)) - (ice/p [:red "✗ Merge failed: " (:err merge-result)]) - (let [winner (if (= merge-strategy :ours) "PR" "master")] - (ice/p [:yellow "Attempting to resolve conflicts with git, preferring changes from " winner "..."]) - (resolve-conflicts (u/versions->artifacts versions) merge-strategy) - ;; Do the commit, now that we've resolved conflicts - (pr-str (p/sh "git" "commit" "--no-edit" "-m" - (str "Merge master into " head-ref-name " for PR #(" pr-number ")" - ", preferring changes from " winner))))) - - (ice/p [:blue "Pushing changes to PR branch..."]) - (ice/p "Result: " (pr-str (p/sh "git" "push" "origin" head-ref-name)))) - - ;; Wait a bit for GitHub to process to avoid a race condition - (Thread/sleep 5000) - - ;; Merge the PR - (ice/p [:blue "Merging PR #" pr-number "..."]) - (let [merge-result (p/sh {:continue true} - "gh" "pr" "merge" (str pr-number) - "--squash" "--delete-branch" - "--repo" "metabase/docs.metabase.github.io")] - (if (zero? (:exit merge-result)) - (ice/p [:green "✓ PR merged successfully!"]) - (ice/p [:red "✗ Merge failed: " [:bold (:err merge-result)]])))) - -(defn- should-pr-win? - "Determine if the current PR should win conflicts based on PR number comparison. - Compares against master, which is what these PRs are always based on." - [current-pr-number] - (let [_ (p/sh "git" "fetch" "origin") - latest-master-commit (-> (p/sh "git" "log" "--oneline" "-1" "origin/master") - :out - str/trim) - ;; Extract PR number from commit message like "[auto] adding content to docs-rc-notes->master (#380)" - master-pr-match (re-find #"#(\d+)" latest-master-commit) - master-pr-number (when master-pr-match (parse-long (second master-pr-match)))] - (ice/p latest-master-commit) - (ice/p [:blue "Current PR: #" current-pr-number]) - (ice/p [:blue "Latest master commit: " latest-master-commit]) - (when master-pr-number - (ice/p [:blue "Latest master PR: #" master-pr-number])) - - (cond - (nil? master-pr-number) - (do (ice/p [:yellow "No PR number found in master, defaulting to PR wins"]) - true) - - (>= current-pr-number master-pr-number) - (do (ice/p [:green "Current PR #" current-pr-number " is newer than master PR #" master-pr-number " - PR wins"]) - true) - - :else - (do (ice/p [:yellow "Current PR #" current-pr-number " is older than master PR #" master-pr-number " - master wins"]) - false)))) - -(defn- checkout-branch! [head-ref-name] - ;; First, discard any local changes that would prevent checkout - (ice/p [:yellow "Discarding local changes..."]) - (p/sh "git" "reset" "--hard" "HEAD") - (p/sh "git" "clean" "-fd") - - ;; Try to checkout the branch - (let [checkout-result (p/shell {:continue true} "git" "checkout" head-ref-name)] - (when-not (zero? (:exit checkout-result)) - ;; Branch doesn't exist locally, create it from origin and force reset - (ice/p [:yellow "Branch doesn't exist locally, creating from origin..."]) - (let [create-result (p/sh {:continue true} "git" "checkout" "-B" head-ref-name (str "origin/" head-ref-name))] - (when-not (zero? (:exit create-result)) - (throw (ex-info (str "Failed to checkout or create branch " head-ref-name) - {:branch head-ref-name - :error (:err create-result) - :babashka/exit 1}))))) - - ;; Force reset to match the remote branch exactly - (ice/p [:yellow "Force resetting to match remote branch..."]) - (p/sh "git" "reset" "--hard" (str "origin/" head-ref-name)))) - -(defn -main [& args] - (println "Merge opertaion running at: " (str (java.time.Instant/now))) - (let [{:keys [versions]} (cli/parse-opts args cli-spec) - versions (u/parse-versions versions) - head-ref-name (u/versions->head-ref-name versions)] - - ;; Ensure we're working with the latest remote state - (ice/p [:blue "Fetching latest from origin..."]) (p/sh "git" "fetch" "origin") - (ice/p [:blue "Checking out branch: " head-ref-name]) (checkout-branch! head-ref-name) - - (let [current-branch (:out (p/sh "git" "branch" "--show-current")) - _ (ice/p [:green "Currently on branch: " (str/trim current-branch)]) - pr-number-view (try (find-pr-view head-ref-name) - (catch Exception e - (ice/p [:red "Error finding pr-number via view: " (ex-message e)]))) - pr-number-list (try (find-pr-list head-ref-name) - (catch Exception e - (ice/p [:red "Error finding pr-number via list: " (ex-message e)]))) - pr-number (or pr-number-view pr-number-list) - merge-strategy (if (should-pr-win? pr-number) :ours :theirs)] - (ice/p [:green "Merging PR #" pr-number ": " head-ref-name " | with strategy: " [:blue merge-strategy]]) - (update-and-merge-pr head-ref-name versions pr-number merge-strategy)))) - -(when (= *file* (System/getProperty "babashka.file")) - (apply -main *command-line-args*)) diff --git a/script/update_or_create_pr.clj b/script/update_or_create_pr.clj deleted file mode 100644 index 1a79f3c76d..0000000000 --- a/script/update_or_create_pr.clj +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env bb -(ns update-or-create-pr - (:require - [babashka.cli :as cli] - [babashka.process :as p] - [cheshire.core :as json] - [clojure.string :as str] - [util :as u])) - -(def cli-spec - {:spec - {:versions {:ref "" - :desc "Comma separated major versions included in this build, eg. \"63,62\"." - :alias :v - :require true} - :annotation {:ref "" - :desc "The annotation to add to the PR." - :default "auto-build"} - :pr-number {:ref "" - :desc "The PR number to update, if it exists." - :default nil} - :update-dirs {:ref "" - :desc "The directories to update in the PR, smart defaults based on the versions." - :default ""}} - :error-fn u/cli-error-fn}) - -(defn existing-pr-num-by-head - "Checks if a PR already exists for the given head branch name." - [head-ref] - (let [raw-data (p/sh {:out :string - :continue true} - "gh" "pr" "list" "--repo" "metabase/docs.metabase.github.io" "--json" "title,number,state,baseRefName,headRefName") - pr-data (-> raw-data :out (json/parse-string true)) - _ (println "→ PR data: " (pr-str pr-data)) - pr-info (first (filter #(= (:headRefName %) head-ref) pr-data))] - (println "→ PR info:" pr-info) - (:number pr-info))) - -(defn- report-pr-body [versions artifact-dirs pr-number] - (str/join "\n" - [(str "Docs pulled from: " - (str/join ", " (map #(str "`release-x." % ".x`") versions))) - "" - "## Updated Directories:" - (str/join "\n" (map #(str "- `" % "`") artifact-dirs)) - "" - (when-not (str/blank? (str pr-number)) - (str "This PR was triggered by: [PR " pr-number "](https://github.com/metabase/metabase/pull/" pr-number ").")) - "" - "> This PR will be merged when the PR that triggered this build is merged."])) - -(defn -main - "Main function to update or create a PR. " - [& args] - (let [{:keys [versions annotation pr-number update-dirs] - :as opts} (cli/parse-opts args cli-spec) - _ (when (or (:help opts) (:h opts)) - (u/pp ["recieved options:" opts]) - (u/show-usage-and-exit cli-spec)) - - versions (u/parse-versions versions) - _ (println "→ Versions in this build: " (str/join ", " versions)) - - head-ref (u/versions->head-ref-name versions) - _ (p/shell "git" "checkout" "-B" head-ref) - - update-dirs (remove str/blank? (str/split update-dirs #",")) - _ (u/pp ["update-dirs" update-dirs]) - - artifact-dirs (concat - update-dirs - (u/versions->artifacts versions)) - _ (doseq [ad artifact-dirs] - (println "Adding" ad "...") - (p/sh {:continue true} "git" "add" ad)) - {diff-exit :exit} (p/shell {:continue true} "git" "diff" "--cached" "--quiet") - pr-title (str "[" annotation "] docs update: " - (str/join ", " (map #(str "v0." %) versions)))] - (if (zero? diff-exit) - (println "→ No changes to commit.") - (do - (println "→ Changes detected, committing...") - (p/shell "git" "commit" "-m" (str "[auto] adding content to " head-ref)) - (p/shell "git" "push" "--force" "origin" head-ref) - (println (str "→ Branch '" head-ref "' updated successfully.")) - (println "→ Checking for existing PR...") - - (if-let [pr-info (existing-pr-num-by-head head-ref)] - (println "✓ PR already exists: #" pr-info) - (do - (println "→ Creating new PR...") - (let [args ["gh" "pr" "create" - "--repo" "metabase/docs.metabase.github.io" - "--title" pr-title - "--body" (report-pr-body versions artifact-dirs pr-number) - "--head" head-ref]] - (println "running: " (str/join " " args)) - (apply p/shell {:continue true} args)))))) - (prn {:versions versions - :head-ref head-ref - :pr-title pr-title - :artifact-dirs artifact-dirs}))) - -(when (= *file* (System/getProperty "babashka.file")) - (apply -main *command-line-args*)) diff --git a/script/util.clj b/script/util.clj index 9b4aa88c8f..29e74d82f5 100644 --- a/script/util.clj +++ b/script/util.clj @@ -62,55 +62,3 @@ (str/replace #"'" "")))) (str/split-lines (slurp (str (fs/expand-home "~/.zshrc")))))))}} cmd)) - - -(def artifacts-to-include []) - -(defn ->artifacts - "Returns a list of directories and files that are considered artifacts - for the given category and release number." - ([target-branch-name] - (let [[category release-num] (categorize-branchname target-branch-name)] - (->artifacts category release-num))) - ([category release-num] - (-> (cond - (= category :master) ["_docs/master"] - - (= (config-docs-version) release-num) - ["_docs/latest" - (str "_docs/v0." release-num)] - - (= category :release) [(str "_docs/v0." release-num)] - :else []) - (concat artifacts-to-include)))) - -(defn parse-versions - "Parse a comma separated list of major versions (eg. \"63,62\") into a seq of - longs, sorted newest first and deduped. Throws on anything that isn't a - positive integer so a typo fails here instead of halfway through a build." - [s] - (let [parts (->> (str/split (str s) #",") - (map str/trim) - (remove str/blank?)) - nums (mapv (fn [part] - (let [n (parse-long part)] - (when-not (and n (pos? n)) - (throw (ex-info (str "Not a major version number: " (pr-str part)) - {:babashka/exit 1 :versions s}))) - n)) - parts)] - (when (empty? nums) - (throw (ex-info "No versions given, expected something like \"63,62\"" - {:babashka/exit 1 :versions s}))) - (vec (distinct (sort > nums))))) - -(defn versions->artifacts - "The union of artifact paths across every release number in `versions`." - [versions] - (into [] (distinct) (mapcat #(->artifacts :release %) versions))) - -(defn versions->head-ref-name - "The PR head branch for a docs update covering `versions`. Sorted newest first - so the name depends on the set of versions, not the order they were listed in." - [versions] - (str "docs-update-" (str/join "-" (map #(str "v" %) (sort > versions)))))