diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml new file mode 100644 index 00000000..6a43440c --- /dev/null +++ b/.github/workflows/build_docs.yml @@ -0,0 +1,43 @@ +name: Build Documentation Site + +# The site is configured with `onBrokenLinks: 'throw'`, so a single bad +# cross-link fails the build. deploy_docs.yml only runs on push to main, which +# meant a broken link was discovered after merging rather than before. This job +# builds the site on pull requests so the PR proves it still builds. + +on: + pull_request: + paths: + - 'documentation/**' + - '.github/workflows/build_docs.yml' + push: + branches: + - main + paths: + - 'documentation/**' + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: '18' + cache: npm + cache-dependency-path: documentation/package-lock.json + + # npm, not yarn: package-lock.json is the lockfile that is committed, and + # there is no yarn.lock. `yarn install` would ignore it and resolve fresh. + - name: Install dependencies + working-directory: documentation + run: npm ci + + - name: Build + working-directory: documentation + run: npm run build diff --git a/.github/workflows/deploy_docs.yml b/.github/workflows/deploy_docs.yml index ee3f5c91..c18f02dc 100644 --- a/.github/workflows/deploy_docs.yml +++ b/.github/workflows/deploy_docs.yml @@ -21,18 +21,21 @@ jobs: with: node-version: '18' + # npm, not yarn: package-lock.json is the lockfile that is committed, and + # there is no yarn.lock. `yarn install` would ignore it and resolve fresh, + # so the deployed site was not built from the pinned dependency tree. - name: Install dependencies run: | cd documentation - yarn install + npm ci - name: Build documentation site run: | cd documentation - yarn build + npm run build - name: Deploy to GitHub Pages uses: peaceiris/actions-gh-pages@v3 with: github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./build \ No newline at end of file + publish_dir: ./documentation/build \ No newline at end of file diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 30a53ea5..8e28f628 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,3 +15,11 @@ jobs: with: options: "--check" src: "." + # Pinned. `psf/black@stable` resolves to whatever Black is newest at the time the job runs, + # so a Black release reformats the world and this check goes red on every open branch with + # nothing in the repository having changed. That is what happened here: main last passed + # this job in November 2025 and fails it today, on 14 files nobody touched. + # + # 25.1.0 is the release the tree is actually formatted for -- verified by running it against + # origin/main, which comes back clean. Bump it deliberately, in a commit that reformats. + version: "25.1.0" diff --git a/CHANGELOG.md b/CHANGELOG.md index b2ed0dfd..b1b1332d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,52 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + + +## [1.2.0] - 2026-08-03 + +### Added +- `tirith platform check`: run an organization's policies against a plan, state or arbitrary JSON + document from CI or a laptop. Masks the document locally, packs it with the terraform source into + an archive, uploads it, creates a StackGuardian run, polls it and reports the verdict as JSON + and/or markdown. The uploaded bundle carries the source under `code/` and a `metadata.json` + describing the repository, the commit and where in the repository `code/` belongs. +- `--fail-on-error` on the local surface too, so evaluating policy files without an account can gate + a merge. Off by default: the local form has always exited 0 either way, and changing that silently + would turn existing green pipelines red. +- `ExitStatus.ERROR_POLICY_FAILED` (3), so a caller can tell "a policy said no" from "tirith could + not tell you". Both surfaces use the same code for the same meaning. Note this applies **only** + with `--fail-on-error`; without it the local form still exits 0 for everything, including a policy + it could not evaluate. + +### Changed +- `cli.main(args=...)` is now honoured. It previously called `parse_args()` with no argument, so + the parameter was ignored and the CLI could only ever read `sys.argv`. + +### Notes +- The local evaluation surface is unchanged, including its single-dash long options. Subcommands + are dispatched before the flat parser sees anything, so `--json` output stays byte-identical. +- No new runtime dependencies: the platform integration is stdlib-only. + +## [1.1.0] - 2026-08-01 + +### Added +- `core`: Policy metadata passthrough — `meta.id`, `meta.name`, `meta.description`, + `meta.severity`, `meta.enforcement`, `meta.tags` and `meta.remediation` now reach the result + document when a policy declares them. Keys that are absent are omitted, so the output of a + policy declaring none of them is unchanged. `{{ var.x }}` substitution works in all of them. + +### Fixed +- `core`: Variable substitution no longer mutates the caller's policy dictionary. Evaluating the + same parsed policy more than once (a policy set, or a retry) previously leaked substituted + values from one evaluation into the next. +- `core`: An unsupported `condition.type` now populates `result` instead of returning without it, + which raised `KeyError` in the pretty printer far from the real cause. +- `core`: Provider errors reported without a `ProviderError` severity are now surfaced instead of + being discarded and `None` evaluated against the condition — a typo'd `operation_type` read as + a genuine policy violation. These are treated as malformed provider calls and are deliberately + not subject to `error_tolerance`. + ## [1.0.5] - 2025-11-19 ### Fixed diff --git a/README.md b/README.md index 786a16e5..62a3acc1 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,41 @@ -[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](code_of_conduct.md) +[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE) +[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](CODE_OF_CONDUCT.md) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=StackGuardian_policy-framework&metric=alert_status&token=4a4d06e73940505edb7fc9d27a7f03b35fbbf23d)](https://sonarcloud.io/summary/new_code?id=StackGuardian_policy-framework) [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=StackGuardian_policy-framework&metric=sqale_rating&token=4a4d06e73940505edb7fc9d27a7f03b35fbbf23d)](https://sonarcloud.io/summary/new_code?id=StackGuardian_policy-framework) [![Slack](https://img.shields.io/badge/Slack-4A154B?style=for-the-badge&logo=slack&logoColor=white)](https://join.slack.com/t/stackguardian-ol78820/shared_invite/zt-2ksag36j9-OjmXqQmyXudgYrV6FmesIQ) [![codecov](https://codecov.io/gh/StackGuardian/tirith/branch/main/graph/badge.svg)](https://codecov.io/gh/StackGuardian/tirith) -# Tirith (StackGuardian Policy Framework) +# Tirith — IaC Governance plugin -## Maintainers - -This project is maintained by [StackGuardian](https://www.linkedin.com/company/stackguardian/). +**Plugin IaC Governance for any pipeline, running anywhere.** Evaluate plans with Tirith, protect +sensitive values, enforce centralised governance, and surface actionable results before +infrastructure changes are applied. +Tirith reads the plan your pipeline already produces — the output of `terraform show -json tfplan` — +checks it against your policies, and exits non-zero so a violating change never reaches `apply`. The +reason it is a plugin rather than an integration is that one policy set then covers every pipeline +you run it from: the same policy files gate a GitHub Actions job, a GitLab job and a laptop, and in +platform mode Tirith rules and Checkov findings come back in one verdict instead of two tools you +have to reconcile by hand. -## A call for contributors - -We are calling for contributors to help build out new features, review pull requests, fix bugs, and maintain overall code quality. If you're interested, please email us at team[at]stackguardian.io or get started by reading the [contributing.md](./CONTRIBUTING.md). - -Tirith scans declarative Infrastructure as Code (IaC) configurations like Terraform against policies defined using JSON. +It is Apache-2.0 and needs no account. Policies are JSON files in your repository, evaluation happens +on your own runner, and nothing is sent anywhere. If you would rather keep policy in one place across +many repositories, `tirith platform check` evaluates against the policies a +[StackGuardian](https://www.stackguardian.io/) organization enforces instead — same document, same +verdict, same exit codes. That mode is optional and is the only part that talks to a network. ## Content - - - [What is Tirith?](#what-is-tirith) - [Features](#features) - [Installation](#installation) - [Usage](#usage) +- [Run it in CI](#run-it-in-ci) +- [Exit codes](#exit-codes) +- [Evaluating against your StackGuardian organization](#evaluating-against-your-stackguardian-organization) - [Example Tirith policies](#example-tirith-policies) + - [error_tolerance](#error_tolerance-and-the-third-outcome) - [Terraform Plan](#terraform-plan-provider) - [Infracost](#infracost-provider) - [StackGuardian Workflow Policy](#stackguardian-workflow-policy-using-sg-workflow-provider) @@ -43,7 +52,10 @@ Tirith scans declarative Infrastructure as Code (IaC) configurations like Terraf ## What is Tirith? -Tirith is a policy framework developed by StackGuardian for enforcing policies on infrastructure configurations such as Terraform, CloudFormation, Kubernetes etc. It simplifies policy creation and enforcement ensuring compliance with infrastructure policies through a user-friendly approach. +Tirith turns a declarative policy — a JSON file, not a program — into a pass or fail verdict on a +concrete infrastructure change. Point it at a terraform plan, a terraform state file, a Kubernetes +manifest, an Infracost breakdown or any JSON document, and it reports which rules passed, which +failed, and on which resource and value. ## Who is the project for? - DevSecOps engineers @@ -66,13 +78,6 @@ Tirith is a policy framework developed by StackGuardian for enforcing policies o - Easily evaluate inputs against policy using pre-defined evaluators like ContainedIn, Equals, RegexMatch etc. - Write your own provider (plugin) by leveraging a highly extensible and pluggable architecture to support any input formats. - ## Installation @@ -82,6 +87,16 @@ This is only a list of approved features that will be included in Tirith over th pip install git+https://github.com/StackGuardian/tirith.git ``` +Pin a tag rather than tracking the default branch, so a CI job cannot change behaviour underneath you: + +``` +pip install "git+https://github.com/StackGuardian/tirith.git@1.0.5" +``` + +`1.0.5` is the newest tag; `git ls-remote --tags https://github.com/StackGuardian/tirith.git` lists +them. Tirith is not on PyPI — `pip install tirith` installs an unrelated project of the same name, so +install from git. Python 3.8 or newer. + ### For developers #### Running the Dev Container @@ -89,8 +104,8 @@ pip install git+https://github.com/StackGuardian/tirith.git - Clone the repository to your local machine: ```bash - git clone - cd + git clone https://github.com/StackGuardian/tirith.git + cd tirith ``` - Start the Docker Engine using docker desktop or CLI. @@ -143,8 +158,7 @@ pip install -e . ``` tirith --version -1.0.0-beta.12 - +tirith 1.2.0 ``` Congratulations! Tirith has been setup in your system @@ -152,7 +166,8 @@ Congratulations! Tirith has been setup in your system ## Usage ``` -usage: tirith [-h] [-policy-path PATH] [-input-path PATH] [--json] [--verbose] [--version] +usage: tirith [-h] [-policy-path PATH] [-input-path PATH] [-var-path PATH] + [-var PATH] [--json] [--verbose] [--fail-on-error] [--version] Tirith (StackGuardian Policy Framework) @@ -160,10 +175,18 @@ options: -h, --help show this help message and exit -policy-path PATH Path containing Tirith policy as code -input-path PATH Input file path + -var-path PATH Variable file path(s) + -var PATH Inline variable(s) --json Only print the result in JSON form (useful for passing output to other programs) --verbose Show detailed logs of from the run + --fail-on-error Exit 3 when a policy fails, instead of 0. Off by default for compatibility. --version show program's version number and exit +Subcommands: + + tirith platform check --help Evaluate against the policies your StackGuardian + organization enforces, rather than local files. + About Tirith: * Abstract away the implementation complexity of policy engine underneath. @@ -171,15 +194,137 @@ About Tirith: * Provide a standard framework for scanning various configurations with granularity. * Provide modularity to enable easy extensibility * Github - https://github.com/StackGuardian/tirith - * Docs - https://docs.stackguardian.io/docs/tirith/overview + * Docs - https://github.com/StackGuardian/tirith#readme +``` + + +## Run it in CI + +### GitHub Actions + +Use [StackGuardian/tirith-iac-governance-action](https://github.com/StackGuardian/tirith-iac-governance-action). +It finds the plan, posts a sticky pull-request comment, creates a check run and sets the job's exit +code: + +```yaml +- run: terraform show -json tfplan > plan.json +- uses: StackGuardian/tirith-iac-governance-action@v2 +``` + +With a `plan.json` in the working directory that is the whole integration — no `with:` block. Add +`with: { fail-on-error: true }` to make a failing policy fail the job, and see the action's own README +for the rest of its inputs. + +### GitLab, or any container-based CI + +There is no GitLab-native equivalent of the action, so you invoke the CLI directly — which is all the +action does underneath. Given an earlier job that saved `plan.json` as an artifact: + +```yaml +policy: + image: python:3.12 + needs: [plan] + script: + - pip install "git+https://github.com/StackGuardian/tirith.git@1.0.5" + - tirith -policy-path .tirith/policies -input-path plan.json --fail-on-error +``` + +Swap the last line for `tirith platform check --workflow-id my-repo --input-path plan.json +--fail-on-error` to use your organization's policies instead of the committed files. Nothing here is +GitLab-specific: any runner that can execute a container and produce a plan works the same way. + +## Exit codes + +| Code | Meaning | +|---|---| +| 0 | Policies passed, or nothing was in scope to gate on | +| 1 | Tirith could not complete the evaluation — bad input, a policy it could not evaluate, unreachable API | +| 2 | Timed out waiting for a StackGuardian run | +| 3 | A policy failed. Only with `--fail-on-error`, on either surface | +| 130 | Interrupted | + +**Gate a CI job with `--fail-on-error`:** + +``` +tirith -policy-path .tirith/policies -input-path plan.json --fail-on-error +echo $? # 3 a policy failed · 1 nothing could be evaluated · 0 everything passed +``` + +Without the flag the exit code is always `0` and the verdict is in the output — that is how the +command has always behaved, and it is left alone so upgrading cannot turn a passing pipeline red. + +**`3` is deliberately not `1`.** `3` means a check ran and said no. `1` means Tirith could not tell you +either way — an unparseable `eval_expression`, an unresolved variable, or a policy whose every check was +skipped. A job that treats every non-zero code alike reports an outage as a policy violation, and +cannot tell a working gate from a broken one. + +One limit worth stating plainly: a *misconfigured* policy — an unsupported `condition.type`, an unknown +`required_provider` — comes back from the engine as an ordinary failed check with no error attached, so +it is indistinguishable from a real violation and exits `3`. It fails closed, which is the safe +direction, but it will point at your infrastructure when the fault is in the policy. + +## Evaluating against your StackGuardian organization + +`tirith platform check` evaluates against the policies your StackGuardian organization enforces, +instead of policy files committed to your repository — so policy lives in one place rather than being +copied into every repository that needs gating. + +``` +export SG_API_TOKEN=sgo_... # an organization token +export SG_ORG=my-org + +tirith platform check --workflow-id my-repo --input-path plan.json --fail-on-error ``` +It masks the document on your machine before anything leaves it, packs it with your terraform source, +uploads it, runs the policies on StackGuardian, and prints the verdict. `--input-path` is optional +when a `plan.json` or `tfplan.json` is in the working directory. + +Common flags: + +| | | +|---|---| +| `--region {eu,us}` | Which StackGuardian region. Default `eu`, or `$SG_REGION` | +| `--api-key -` | Read the key from stdin instead of the environment | +| `--plan-file tfplan` | The binary plan from `terraform plan -out=`, rendered through `terraform show -json` in memory. Use `--input-path` if you already have the JSON | +| `--state-path` / `--infracost-path` | Add a state document or a cost breakdown to the evaluation | +| `--no-source` | Do not upload the terraform source. Discovery still looks in `--source-dir` for the plan | +| `--fail-on-error` | Exit `3` when a policy fails, instead of `0` | +| `--output-json` / `--output-markdown` | Write the verdict to files for a later CI step | + +`--api-url` overrides `--region` for a self-hosted or dedicated host. Every flag is in +[docs/platform-check.md](docs/platform-check.md) or `tirith platform check --help`. + +Running this from GitHub Actions? Use [the action](#github-actions) instead — it wires up the plan +discovery, the sticky pull-request comment, the check run and the exit codes for you. + ## Example Tirith policies [Examples using various providers](tests/providers) +### `error_tolerance`, and the third outcome + +Every `condition` takes an `error_tolerance`, and it appears in most of the examples below without +being explained. It is a severity threshold for *problems reading the input*, not for policy failures: + +- **`0`** — anything the provider could not read is an error, and the check **fails**. +- **`1` or higher** — a problem whose severity is at or below the tolerance is *skipped* instead. A + missing attribute has severity 2, so `error_tolerance: 2` turns "this key is not in the plan" from a + failure into a non-answer. + +That third outcome is why some sample output below shows `"passed": null` rather than `true` or +`false` — the check did not pass and did not fail, it never ran. A skipped check is then **removed from +`eval_expression`** before it is evaluated, because `None` is falsy in Python and leaving it in would +silently read as a failure. + +One consequence worth knowing before using it: a policy whose every check is skipped has evaluated +nothing at all, and reports `"final_result": null` rather than `true` or `false`. With +`--fail-on-error` that exits **1**, not 0 and not 3 — a check that looked at nothing is not a pass, and +it is not a violation either. Keep the tolerance at `0` if you would rather such a policy fail outright. + ### Terraform plan provider
+Terraform plan provider — example policies and output #### Example 1: VPC and EC2 instance policy @@ -311,7 +456,7 @@ Policy: } } ], - "eval_expression": "check1 && check11 && check111 & check2 & check22" + "eval_expression": "check1 && check22" } ``` @@ -489,7 +634,7 @@ JSON Output: } ], "errors": [], - "eval_expression": "check1 && check11 && check111 & check2 & check22" + "eval_expression": "check1 && check22" } ``` @@ -497,6 +642,7 @@ JSON Output: ### Infracost Provider
+Infracost Provider — example policies and output Cost control policy @@ -648,6 +794,7 @@ JSON Output: ### StackGuardian Workflow Policy (using SG workflow provider)
+StackGuardian Workflow Policy (using SG workflow provider) — example policies and output - Terraform Workflow should require an approval to create or destroy resources ```json @@ -800,6 +947,7 @@ JSON Output: ### JSON
+JSON — example policies and output Example Policy ```json @@ -995,14 +1143,16 @@ JSON Output ], "errors": [], "eval_expression": "check1 && check2 && check3 && check4 && check5" +} ```
### Kubernetes
+Kubernetes — example policies and output Kubernetes (using Kubernetes provider) -#### Example 1 +#### Example - Make sure that all pods have a liveness probe defined ```json @@ -1029,71 +1179,9 @@ Kubernetes (using Kubernetes provider) "eval_expression": "!kinds_have_null_liveness_probe" } ``` -#### Example 2 -Example Policy: +Example output: -```json -{ - "meta": { - "version": "v1", - "required_provider": "stackguardian/kubernetes" - }, - "evaluators": [ - { - "id": "kinds_have_null_liveness_probe", - "provider_args": { - "operation_type": "attribute", - "kubernetes_kind": "Pod", - "attribute_path": "spec.containers.*.livenessProbe" - }, - "condition": { - "type": "Contains", - "value": null, - "error_tolerance": 2 - } - } - ], - "eval_expression": "!kinds_have_null_liveness_probe" -} -``` - -Example Input: - -```yml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: wfs-demp-wfs-demo - labels: - helm.sh/chart: wfs-demo-0.1.0 - app.kubernetes.io/name: wfs-demo - app.kubernetes.io/instance: wfs-demp - app.kubernetes.io/version: "1.16.0" - app.kubernetes.io/managed-by: Helm ---- -# Source: wfs-demo/templates/user-acces.yaml -apiVersion: rbac.authorization.k8s.io/v1 -... - - name: wget - image: busybox - command: ['wget'] - args: ['wfs-demp-wfs-demo:80'] - livenessProbe: - exec: - command: - - cat - - /tmp/healthy - initialDelaySeconds: 5 - periodSeconds: 5 - restartPolicy: Never - -``` - -Output: -![](docs/kubernetes_example.gif) - -JSON Output: ```json { "meta": { @@ -1121,30 +1209,8 @@ JSON Output: ```
- - - + + ## Getting Started This is a short getting started guide for Tirith. We will take a look on how we can use Tirith to guardrail a JSON input. @@ -1265,7 +1331,9 @@ Final expression used: ## Want to contribute? -If you're interested, please email us at team[at]stackguardian.io or get started by reading the [contributing.md](./CONTRIBUTING.md). +We are calling for contributors to help build out new features, review pull requests, fix bugs, and +maintain overall code quality. Email us at team[at]stackguardian.io, or get started by reading +[contributing.md](./CONTRIBUTING.md). ### Getting an issue assigned @@ -1297,8 +1365,17 @@ Wanna submit a feedback? It's as simple as writing and posting it in the Your feedback will help us improve

+## Maintainers + +This project is maintained by [StackGuardian](https://www.linkedin.com/company/stackguardian/). + ## Support +Open an [issue](https://github.com/StackGuardian/tirith/issues) for a bug or a question about policy +authoring. For anything specific to a StackGuardian organization — enforcement scope, a run that +errored, an API key — contact StackGuardian support instead, since that needs account context this +repository has no access to. + ## License Apache License 2.0 diff --git a/docs/platform-check.md b/docs/platform-check.md new file mode 100644 index 00000000..54eae4ec --- /dev/null +++ b/docs/platform-check.md @@ -0,0 +1,255 @@ +# `tirith platform check` + +Evaluate a terraform plan, state document or cost breakdown against the policies your StackGuardian +organization enforces, from any CI system or from a laptop. + +The [GitHub Action](https://github.com/StackGuardian/tirith-iac-governance-action) is a thin wrapper +around this command. Use the action on GitHub; use this directly anywhere else — GitLab CI, a +Makefile, a local shell. + +## What it does + +1. **Masks the document on your machine**, before anything is uploaded. Values terraform marked + sensitive are replaced with `__SG_REDACTED__`, root `variables` are dropped, and `prior_state` is + removed. `json` and `kubernetes` documents are *not* masked — there is no schema that says which + fields are secret. +2. **Packs** the masked documents with your terraform source into a `tar.gz`, excluding `.git`, + `.terraform`, `*.tfstate*` and anything matched by `.gitignore`. `--source-dir ""` sends documents + only. An oversized tree degrades to documents-only rather than failing. +3. **Uploads it** to the workflow's artifact directory and creates a StackGuardian workflow run. +4. **Polls** the run and prints the verdict, optionally as JSON and markdown for a later CI step. + +Committed source ships as written: a secret hardcoded in HCL reaches the platform even though the +plan was masked. `--source-dir ""` is the opt-out. + +## Credentials + +`--api-key` / `$SG_API_TOKEN` and `--org` / `$SG_ORG`. The key should be an **organization** (`sgo_`) +token — `sgu_` keys are non-functional for SSO-group-only users, and are warned about rather than +rejected, so the symptom is a later 403. + +`--api-key -` reads the key from stdin, which keeps it out of the process table and out of shell +history: + + echo "$SG_TOKEN" | tirith platform check --api-key - --workflow-id infra + +## Workflow identity + +`--workflow-id` names the StackGuardian workflow, and is created on first use. `--workflow-group` +defaults to `default`. + +Two things worth knowing before choosing an id: + +* Runs on one workflow **serialize** while another is pending. A matrix that shares an id becomes a + queue, so give each leg its own. +* `--artifact-tag` namespaces the uploaded bundle. Two runs of the same workflow with the same tag + and the same commit reuse one name, which is fine; different commits never collide. + +## What the bundle contains + +The archive uploaded to the workflow's artifact directory has a fixed layout. It is a contract: the +step reads its inputs out of it, and other systems read it to see the code a verdict came from. + +``` +plan.json the masked terraform plan +tfstate.json the masked state, if one was supplied +infracost.json the cost breakdown, if one was supplied +metadata.json what this bundle is +code/ the terraform source, if any was packed +``` + +Documents sit at the **root**; the source sits under **`code/`**. `code/` is a path prefix rather than +a directory entry, so it is absent entirely when no source was packed — `metadata.json` says which, +and why. + +### metadata.json + +Field names are `snake_case`, matching the other JSON this tool authors and the `plan.json` beside it. +Everything about the repository is nullable, because a local run has no repository to describe and a +fabricated one would be worse than an honest `null`. + +```json +{ + "schema_version": 1, + "generator": {"name": "tirith", "version": "1.2.0"}, + "created_at": "2026-08-12T09:14:03Z", + "input_kind": "terraform_plan", + "origin": {"kind": "ci", "trigger_type": "tirith", "ci_run_url": "https://github.com/acme/infra/actions/runs/1"}, + "repository": { + "provider": "github", + "host": "github.com", + "url": "https://github.com/acme/infra", + "ref": "feat/rds", + "commit": "9f2c1ab5e0d34c7f8b1a2d3e4f506172", + "change_request": {"id": "412", "url": "https://github.com/acme/infra/pull/412", "target_ref": null} + }, + "code": { + "present": true, + "prefix": "code/", + "repo_path": "infra/prod", + "repo_path_from": "git_root", + "files": 37, + "skipped": 5, + "absent_reason": null + }, + "documents": {"plan": "plan.json", "state": null, "infracost": null}, + "masking": {"redactions": 12, "marker": "__SG_REDACTED__", "documents_are_masked": true}, + "workflow": {"org": "acme", "group": "default", "id": "infra-prod", "artifact_tag": "default"} +} +``` + +The fields worth understanding before writing a consumer: + +**`code.repo_path`** is the one that cannot be recovered any other way. Members are named relative to +`--source-dir`, so the path *within the repository* is destroyed at pack time. `--source-dir infra/prod` +means `code/main.tf` belongs at `infra/prod/main.tf`. It is `""` for the repository root — not `"."`, +and not `null`, so joining still works and it stays distinguishable from "we could not tell", which is +`null`. `code.repo_path_from` is `"flag"` when `--repo-path` declared it and `"git_root"` when it was +inferred from the enclosing checkout: for anything about to write into a repository, declared and +inferred are not the same confidence. + +**`code.present`** means "there are members under `code/`", not "a source directory was requested". A +tree whose every file was excluded produces `present: false` with `files: 0`, so the metadata and the +tar can never disagree. `absent_reason` is `not_requested` (`--no-source`), `too_large` (the tree was +dropped so the check could still run) or `empty_after_excludes`. + +**`masking`** exists so a consumer knows not to feed these documents to terraform. A `tfstate.json` +full of `__SG_REDACTED__` looks exactly like state and would destroy infrastructure if applied. + +**`repository.provider`** is sniffed from the host, independently of `origin` — a GitHub Actions job +can check out a GitLab repository. A host we do not recognise is `"unknown"` with `host` still set, +rather than guessed. Any credential in the URL is stripped before it is written. + +**`schema_version`** is a single integer, bumped only by a breaking change; added fields do not bump it. +On a higher version than you know, read what you recognise and do not act destructively — in particular +do not write files back using a `code.repo_path` from a schema you do not understand. A bundle with no +`metadata.json` at all predates this and should be treated as undescribed rather than invalid. + +## Exit codes + +| Code | Meaning | +|---|---| +| 0 | Policies passed, or nothing was in scope | +| 1 | Could not complete the check | +| 2 | Timed out waiting for the run | +| 3 | A policy failed — only with `--fail-on-error` | +| 130 | Interrupted | + +`3` exists so a caller can distinguish "your infrastructure violates a policy" from "Tirith could not +reach the platform". Without `--fail-on-error` a policy failure still exits `0`, and the verdict is +in `--output-json`. + +## Full flag reference + +``` +usage: tirith platform check [-h] [--api-key API_KEY] [--org ORG] + [--region {eu,us}] [--api-url API_URL] + [--dashboard-url DASHBOARD_URL] + --workflow-id WORKFLOW_ID + [--workflow-group WORKFLOW_GROUP] + [--terraform-version TERRAFORM_VERSION] + [--repo-url REPO_URL] [--repo-ref REPO_REF] + [--repo-path REPO_PATH] + [--step-template-id STEP_TEMPLATE_ID] + [--input-path INPUT_PATH] [--plan-file PLAN_FILE] + [--terraform-bin TERRAFORM_BIN] + [--input-kind {terraform_plan,terraform_state,kubernetes,json}] + [--state-path STATE_PATH] + [--infracost-path INFRACOST_PATH] + [--source-dir SOURCE_DIR] [--no-source] + [--sha SHA] [--artifact-tag ARTIFACT_TAG] + [--trigger-details-json TRIGGER_DETAILS_JSON] + [--trigger-details-file TRIGGER_DETAILS_FILE] + [--timeout TIMEOUT] [--output-json OUTPUT_JSON] + [--output-markdown OUTPUT_MARKDOWN] + [--comment-marker COMMENT_MARKER] + [--markdown-limit MARKDOWN_LIMIT] + [--fail-on-error] + +Masks the document, packs it with the terraform source into an archive, +uploads it, runs the policies on StackGuardian and reports the verdict. + +options: + -h, --help show this help message and exit + +identity: + --api-key API_KEY API key, or '-' to read it from stdin. Default: + $SG_API_TOKEN + --org ORG Organization name. Default: $SG_ORG + --region {eu,us} StackGuardian region, setting both URLs at once. + Default: $SG_REGION or eu. + --api-url API_URL API base URL, with or without /api/v1. Overrides + --region; needed only for a self-hosted install or a + dedicated host. Default: $SG_BASE_URL + --dashboard-url DASHBOARD_URL + Dashboard base URL, used to build run links. Inferred + from --api-url when it names a known region. + +workflow: + --workflow-id WORKFLOW_ID + Slug identifying the workflow. Created if absent. + Letters, digits, '-' and '_' only. + --workflow-group WORKFLOW_GROUP + Workflow group. Created if absent. + --terraform-version TERRAFORM_VERSION + Stored on the workflow at creation. + --repo-url REPO_URL Source repository URL, recorded on the workflow at + creation so it links back to the code. + --repo-ref REPO_REF Branch, tag or commit, recorded alongside --repo-url. + --repo-path REPO_PATH + Path of --source-dir within the repository, recorded + in the bundle's metadata.json so a consumer knows + where code/ belongs. Inferred from the enclosing git + checkout if omitted. + --step-template-id STEP_TEMPLATE_ID + Override the policy-evaluation step template. Omit to + use the platform's own default. + +inputs: + --input-path INPUT_PATH + Document to evaluate. Defaults to whichever of + plan.json or tfplan.json is in --source-dir. + --plan-file PLAN_FILE + Binary plan from `terraform plan -out=`. Rendered with + `show -json` in memory, so no unmasked plan JSON is + written to disk. Use --input-path if you already have + the JSON. + --terraform-bin TERRAFORM_BIN + terraform/tofu binary for --plan-file. Auto-detected, + preferring the real binary over a CI wrapper. + --input-kind {terraform_plan,terraform_state,kubernetes,json} + --state-path STATE_PATH + Optional terraform state, masked before upload. + --infracost-path INFRACOST_PATH + Optional `infracost breakdown --format json`. + --source-dir SOURCE_DIR + Terraform source to pack alongside the documents. + --no-source Send only the documents. Discovery still looks in + --source-dir (or .) for the plan. + +run: + --sha SHA Commit SHA, used to namespace the uploaded archive. + --artifact-tag ARTIFACT_TAG + Namespaces the archive within a commit. Needed only + when one workflow evaluates the same commit more than + once -- a plan phase and a state phase, or matrix legs + sharing a workflow. + --trigger-details-json TRIGGER_DETAILS_JSON + JSON object describing what triggered this run. + --trigger-details-file TRIGGER_DETAILS_FILE + File containing that JSON object. + --timeout TIMEOUT Seconds to wait for the run. Default: 1800 + +output: + --output-json OUTPUT_JSON + Write the result document here. + --output-markdown OUTPUT_MARKDOWN + Write a markdown report here. + --comment-marker COMMENT_MARKER + Opaque first line of the markdown, for stickiness. + --markdown-limit MARKDOWN_LIMIT + Truncate the markdown to this length. + --fail-on-error Exit non-zero when a policy fails. An unreachable + platform or a run that produced no verdict always + exits non-zero regardless of this flag. +``` diff --git a/documentation/docs/tirith-policies/tirith-policy-cookbook.md b/documentation/docs/tirith-policies/tirith-policy-cookbook.md new file mode 100644 index 00000000..aac36e8f --- /dev/null +++ b/documentation/docs/tirith-policies/tirith-policy-cookbook.md @@ -0,0 +1,529 @@ +--- +id: tirith-policy-cookbook +title: Policy Cookbook +sidebar_label: Policy Cookbook +description: Complete, runnable Tirith policies for common real-world checks, each shown with its input and the verdict it produces. +keywords: + - tirith +site_name: Tirith +slug: tirith-policy-cookbook/ +--- + +Every recipe on this page is complete: copy the policy and the input into files, run the command shown, and you will get the output shown. All commands use `--fail-on-error` so the exit code carries the verdict — `0` pass, `3` fail, `1` when the run could not produce a verdict (see the [exit code table](./tirith-policy-reference.md#outcomes-and-exit-codes) and the [CLI reference](../tirith-usage/cli-reference.md)). + +Field-by-field schema details are in the [Policy Reference](./tirith-policy-reference.md); condition semantics in the [evaluator reference](../tirith-reference/evaluators.md); provider operations in the [provider documentation](../tirith-providers/overview.md). + +## Forbid unapproved instance types + +Every `aws_instance` in a Terraform plan must use an instance type from an approved list. `ContainedIn` checks each extracted value against the list, and the check passes only if **all** instances pass — so one oversized instance fails the whole policy. + +```json title="policy.json" +{ + "meta": { + "version": "v1", + "required_provider": "stackguardian/terraform_plan", + "id": "allowed-instance-types", + "name": "Only approved EC2 instance types", + "description": "Every aws_instance in the plan must use an instance type from the approved list.", + "severity": "HIGH" + }, + "evaluators": [ + { + "id": "instance_type_is_approved", + "description": "aws_instance.instance_type must be one of the approved types", + "provider_args": { + "operation_type": "attribute", + "terraform_resource_type": "aws_instance", + "terraform_resource_attribute": "instance_type" + }, + "condition": { + "type": "ContainedIn", + "value": ["t3.micro", "t3.small", "t3.medium"] + } + } + ], + "eval_expression": "instance_type_is_approved" +} +``` + +The input is a Terraform plan in JSON form (`terraform show -json plan.out > input.json`). This trimmed-down plan has one compliant and one non-compliant instance: + +```json title="input.json" +{ + "format_version": "1.2", + "terraform_version": "1.5.7", + "resource_changes": [ + { + "address": "aws_instance.web", + "type": "aws_instance", + "name": "web", + "change": { + "actions": ["create"], + "after": { + "instance_type": "t3.small", + "tags": { "Environment": "prod" } + } + } + }, + { + "address": "aws_instance.batch", + "type": "aws_instance", + "name": "batch", + "change": { + "actions": ["create"], + "after": { + "instance_type": "m5.24xlarge", + "tags": { "Environment": "prod" } + } + } + } + ] +} +``` + +```bash +tirith --fail-on-error -policy-path policy.json -input-path input.json +``` + +```text +Check: instance_type_is_approved + FAILED + 1. PASSED: Found `"t3.small"` inside `["t3.medium", "t3.micro", "t3.small"]` + 2. FAILED: Failed to find `"m5.24xlarge"` inside `["t3.medium", "t3.micro", "t3.small"]` + +Passed: 0 Failed: 1 Skipped: 0 + +Final expression used: +-> instance_type_is_approved +✘ Failed final evaluation +``` + +Exit code: `3`. + +## Require an Environment tag on every resource + +With `terraform_resource_type` set to `"*"`, the check runs against every resource in the plan. The dotted attribute path reaches into the `tags` map, and `RegexMatch` restricts the value to an allowed set. + +```json title="policy.json" +{ + "meta": { + "version": "v1", + "required_provider": "stackguardian/terraform_plan", + "id": "require-environment-tag", + "name": "Every resource carries an Environment tag", + "description": "Every resource in the plan must be tagged with Environment set to dev, staging, or prod." + }, + "evaluators": [ + { + "id": "environment_tag_is_valid", + "provider_args": { + "operation_type": "attribute", + "terraform_resource_type": "*", + "terraform_resource_attribute": "tags.Environment" + }, + "condition": { + "type": "RegexMatch", + "value": "^(dev|staging|prod)$" + } + } + ], + "eval_expression": "environment_tag_is_valid" +} +``` + +```json title="input.json" +{ + "format_version": "1.2", + "terraform_version": "1.5.7", + "resource_changes": [ + { + "address": "aws_s3_bucket.artifacts", + "type": "aws_s3_bucket", + "name": "artifacts", + "change": { + "actions": ["create"], + "after": { + "bucket": "team-artifacts", + "tags": { "Environment": "prod" } + } + } + }, + { + "address": "aws_instance.web", + "type": "aws_instance", + "name": "web", + "change": { + "actions": ["create"], + "after": { + "instance_type": "t3.small", + "tags": { "Environment": "staging" } + } + } + } + ] +} +``` + +```bash +tirith --fail-on-error -policy-path policy.json -input-path input.json +``` + +```text +Check: environment_tag_is_valid + PASSED + 1. PASSED: `"prod"` matches regex pattern `"^(dev|staging|prod)$"` + 2. PASSED: `"staging"` matches regex pattern `"^(dev|staging|prod)$"` + +Passed: 1 Failed: 0 Skipped: 0 + +Final expression used: +-> environment_tag_is_valid +✔ Passed final evaluator +``` + +Exit code: `0`. + +A resource with no `tags.Environment` at all fails rather than slipping through. Against an input whose only resource has no tags: + +```text +Check: environment_tag_is_valid + FAILED + 1. FAILED: attribute: 'tags.Environment' is not found + +Passed: 0 Failed: 1 Skipped: 0 + +Final expression used: +-> environment_tag_is_valid +✘ Failed final evaluation +``` + +Exit code: `3`. (A missing attribute is a severity-2 provider error; the default `error_tolerance` of 0 turns it into a failure. The [last recipe](#tolerate-a-missing-key) shows how to skip instead.) + +## Block security group ingress from 0.0.0.0/0 + +A public-ingress check. The attribute path `ingress.*.cidr_blocks` extracts the `cidr_blocks` list of **each** ingress rule, and `NotContains` requires that none of those lists contain `0.0.0.0/0`. + +```json title="policy.json" +{ + "meta": { + "version": "v1", + "required_provider": "stackguardian/terraform_plan", + "id": "no-public-ingress", + "name": "No security group ingress from 0.0.0.0/0", + "description": "No ingress rule of any aws_security_group may allow traffic from 0.0.0.0/0.", + "severity": "HIGH", + "remediation": "Restrict the CIDR range or reference another security group instead." + }, + "evaluators": [ + { + "id": "no_public_cidr_in_ingress", + "provider_args": { + "operation_type": "attribute", + "terraform_resource_type": "aws_security_group", + "terraform_resource_attribute": "ingress.*.cidr_blocks" + }, + "condition": { + "type": "NotContains", + "value": "0.0.0.0/0" + } + } + ], + "eval_expression": "no_public_cidr_in_ingress" +} +``` + +```json title="input.json" +{ + "format_version": "1.2", + "terraform_version": "1.5.7", + "resource_changes": [ + { + "address": "aws_security_group.web", + "type": "aws_security_group", + "name": "web", + "change": { + "actions": ["create"], + "after": { + "name": "web-sg", + "ingress": [ + { + "from_port": 443, + "to_port": 443, + "protocol": "tcp", + "cidr_blocks": ["10.0.0.0/8"] + }, + { + "from_port": 22, + "to_port": 22, + "protocol": "tcp", + "cidr_blocks": ["0.0.0.0/0"] + } + ] + } + } + } + ] +} +``` + +```bash +tirith --fail-on-error -policy-path policy.json -input-path input.json +``` + +```text +Check: no_public_cidr_in_ingress + FAILED + 1. PASSED: Did not find 0.0.0.0/0 inside ['10.0.0.0/8'] + 2. FAILED: Found `"0.0.0.0/0"` inside `["0.0.0.0/0"]` + +Passed: 0 Failed: 1 Skipped: 0 + +Final expression used: +-> no_public_cidr_in_ingress +✘ Failed final evaluation +``` + +Exit code: `3`. The port-443 rule scoped to `10.0.0.0/8` passes; the SSH rule open to the world fails the policy. + +## Cap the estimated monthly cost + +Uses the `stackguardian/infracost` provider against an [Infracost](https://www.infracost.io/) breakdown (`infracost breakdown --path . --format json > input.json`). The `total_monthly_cost` operation sums the monthly cost of the matched resources; `["*"]` matches all of them. + +```json title="policy.json" +{ + "meta": { + "version": "v1", + "required_provider": "stackguardian/infracost", + "id": "monthly-cost-ceiling", + "name": "Monthly cost stays under the ceiling", + "description": "The estimated total monthly cost of all resources must not exceed 500 USD." + }, + "evaluators": [ + { + "id": "total_monthly_cost_under_ceiling", + "provider_args": { + "operation_type": "total_monthly_cost", + "resource_type": ["*"] + }, + "condition": { + "type": "LessThanEqualTo", + "value": 500 + } + } + ], + "eval_expression": "total_monthly_cost_under_ceiling" +} +``` + +```json title="input.json" +{ + "version": "0.2", + "currency": "USD", + "projects": [ + { + "name": "main", + "breakdown": { + "resources": [ + { + "name": "aws_instance.web", + "monthlyCost": "301.44" + }, + { + "name": "aws_db_instance.app", + "monthlyCost": "109.86" + }, + { + "name": "aws_s3_bucket.artifacts", + "monthlyCost": "2.30" + } + ] + } + } + ] +} +``` + +```bash +tirith --fail-on-error -policy-path policy.json -input-path input.json +``` + +```text +Check: total_monthly_cost_under_ceiling + PASSED + 1. PASSED: `413.6` is less than equal to `500` + +Passed: 1 Failed: 0 Skipped: 0 + +Final expression used: +-> total_monthly_cost_under_ceiling +✔ Passed final evaluator +``` + +Exit code: `0`. To limit the sum to particular resource types instead, list them: `"resource_type": ["aws_instance", "aws_db_instance"]`. + +## Tolerate a missing key + +By default a value the provider cannot find fails the check. `error_tolerance` turns "the data is absent" into a **skip** instead — here the logging level is validated only when a `logging` block exists at all. Encryption, by contrast, gets no tolerance: its absence must fail. + +```json title="policy.json" +{ + "meta": { + "version": "v1", + "required_provider": "stackguardian/json", + "id": "encryption-and-optional-logging", + "name": "Encryption required, logging checked when configured", + "description": "Encryption must be enabled. The logging level is validated only when the logging block exists." + }, + "evaluators": [ + { + "id": "encryption_enabled", + "provider_args": { + "operation_type": "get_value", + "key_path": "spec.encryption.enabled" + }, + "condition": { + "type": "Equals", + "value": true + } + }, + { + "id": "logging_level_is_valid", + "provider_args": { + "operation_type": "get_value", + "key_path": "spec.logging.level" + }, + "condition": { + "type": "ContainedIn", + "value": ["INFO", "WARN", "ERROR"], + "error_tolerance": 2 + } + } + ], + "eval_expression": "encryption_enabled && logging_level_is_valid" +} +``` + +```json title="input.json" +{ + "spec": { + "encryption": { + "enabled": true + } + } +} +``` + +```bash +tirith --fail-on-error -policy-path policy.json -input-path input.json +``` + +```text +Check: encryption_enabled + PASSED + 1. PASSED: `true` is equal to `true` + +Check: logging_level_is_valid + SKIPPED + 1. SKIPPED: key_path: `spec.logging.level` is not found (severity: 2) + +Passed: 1 Failed: 0 Skipped: 1 + +Final expression used: +-> encryption_enabled && logging_level_is_valid +✔ Passed final evaluator +``` + +Exit code: `0`. The skipped check is removed from `eval_expression` — the expression effectively becomes `encryption_enabled` — so the policy passes. The missing `key_path` is a severity-2 provider error; `"error_tolerance": 2` absorbs it. If the input *does* contain `spec.logging.level`, the value is validated normally and `DEBUG` would fail the policy. + +One consequence to be aware of: if **every** check in the expression is skipped, the final verdict is neither pass nor fail. Running only the tolerant check against the same input: + +```text +Check: logging_level_is_valid + SKIPPED + 1. SKIPPED: key_path: `spec.logging.level` is not found (severity: 2) + +Passed: 0 Failed: 0 Skipped: 1 + +Final expression used: +-> logging_level_is_valid += Skipped final evaluator +``` + +Exit code: `1` — with `--fail-on-error`, an all-skipped run counts as an error, not a pass, because nothing was actually verified. Without the flag the exit code is `0`, like every other outcome. + +## Parameterize the policy with variables + +The same cost-ceiling policy, with the limit supplied at run time. A variable reference must be the entire string value; it is replaced with the variable's JSON value, so a number stays a number. + +```json title="policy.json" +{ + "meta": { + "version": "v1", + "required_provider": "stackguardian/infracost", + "id": "parameterized-cost-ceiling", + "name": "Monthly cost stays under a configurable ceiling", + "description": "The estimated total monthly cost must not exceed the ceiling supplied as a variable." + }, + "evaluators": [ + { + "id": "cost_under_ceiling", + "provider_args": { + "operation_type": "total_monthly_cost", + "resource_type": ["*"] + }, + "condition": { + "type": "LessThanEqualTo", + "value": "{{ var.max_monthly_cost }}" + } + } + ], + "eval_expression": "cost_under_ceiling" +} +``` + +```json title="variables.json" +{ + "max_monthly_cost": 300 +} +``` + +Run against the same `input.json` as the previous cost recipe (total: 413.60): + +```bash +tirith --fail-on-error -policy-path policy.json -input-path input.json -var-path variables.json +``` + +```text +Check: cost_under_ceiling + FAILED + 1. FAILED: `413.6` is not less than or equal to `300` + +Passed: 0 Failed: 1 Skipped: 0 + +Final expression used: +-> cost_under_ceiling +✘ Failed final evaluation +``` + +Exit code: `3`. + +An inline `-var` overrides the variable file: + +```bash +tirith --fail-on-error -policy-path policy.json -input-path input.json \ + -var-path variables.json -var 'max_monthly_cost=1000' +``` + +```text +Check: cost_under_ceiling + PASSED + 1. PASSED: `413.6` is less than equal to `1000` + +Passed: 1 Failed: 0 Skipped: 0 + +Final expression used: +-> cost_under_ceiling +✔ Passed final evaluator +``` + +Exit code: `0`. If a referenced variable is not supplied at all, the policy is not evaluated: the output reports `Variables not found: max_monthly_cost` and there is no verdict (exit `1` with `--fail-on-error`). The full substitution rules are in the [Policy Reference](./tirith-policy-reference.md#variables). diff --git a/documentation/docs/tirith-policies/tirith-policy-reference.md b/documentation/docs/tirith-policies/tirith-policy-reference.md new file mode 100644 index 00000000..6d271a3b --- /dev/null +++ b/documentation/docs/tirith-policies/tirith-policy-reference.md @@ -0,0 +1,236 @@ +--- +id: tirith-policy-reference +title: Policy Reference +sidebar_label: Policy Reference +description: Field-by-field reference for the Tirith policy file format, including every key, its type, its default, and its failure behavior. +keywords: + - tirith +site_name: Tirith +slug: tirith-policy-reference/ +--- + +A Tirith policy is a single JSON document with exactly three top-level keys. It is evaluated against an input document passed to the CLI with `-input-path` (see the [CLI reference](../tirith-usage/cli-reference.md)). The policy file itself is always JSON; the input file is parsed as JSON unless its name ends in `.yaml` or `.yml`, in which case it is parsed as YAML (a multi-document YAML file becomes a list of documents). + +```json title="policy.json" +{ + "meta": { + "version": "v1", + "required_provider": "stackguardian/json" + }, + "evaluators": [ + { + "id": "can_post", + "provider_args": { + "operation_type": "get_value", + "key_path": "verb" + }, + "condition": { + "type": "Equals", + "value": "POST" + } + } + ], + "eval_expression": "can_post" +} +``` + +Unknown keys, at any level, are ignored. + +## Top-level keys + +| Key | Required | Type | Description | +|---|---|---|---| +| `meta` | yes | object | Policy metadata. Selects the provider; everything else is informational. | +| `evaluators` | yes | array of objects | The checks. Each one extracts values from the input and compares them against a condition. | +| `eval_expression` | yes | string | Boolean expression over evaluator `id`s that produces the final verdict. | + +If `meta`, `evaluators`, or `eval_expression` is missing, the run aborts before producing a verdict: the CLI prints `ERROR` and exits with code `1` (with or without `--fail-on-error`). + +## `meta` + +| Key | Required | Type | Default | Behavior | +|---|---|---|---|---| +| `required_provider` | effectively yes | string | `"core"` | Selects the provider used by every evaluator in the policy. See [below](#metarequired_provider). | +| `version` | no | string | none | Not interpreted. Always echoed into the result `meta` (as `null` when absent). | +| `id` | no | any (conventionally string) | none | Not interpreted. Echoed verbatim into the result `meta` only when present. | +| `name` | no | any (conventionally string) | none | Same as `id`. | +| `description` | no | any (conventionally string) | none | Same as `id`. | +| `severity` | no | any (conventionally string) | none | Same as `id`. | +| `enforcement` | no | any (conventionally string) | none | Same as `id`. See [below](#metaenforcement). | +| `tags` | no | any (conventionally array of strings) | none | Same as `id`. | +| `remediation` | no | any (conventionally string) | none | Same as `id`. | + +An empty `meta` object (`"meta": {}`) is accepted; the policy then falls back to the default provider, which fails every check (see next section). + +### `meta.required_provider` + +The registered providers are: + +- `stackguardian/terraform_plan` +- `stackguardian/infracost` +- `stackguardian/sg_workflow` +- `stackguardian/json` +- `stackguardian/kubernetes` + +Each provider defines its own `provider_args`; see the [provider documentation](../tirith-providers/overview.md). + +When `required_provider` is absent it defaults to `"core"`, and no provider named `core` is registered. An unregistered provider name — the default included — is **not** a hard error: every evaluator in the policy simply receives no values and fails with the message `Could not find input value`. The final verdict is a failure (exit code `3` under `--fail-on-error`), which can be mistaken for a genuine policy violation. Always set `required_provider` explicitly. + +### `meta.enforcement` + +The open-source engine does **not** interpret this field. There is no list of accepted values, no validation, and no warning: any value — `hard_mandatory`, `soft_mandatory`, or any other string — is copied verbatim into the result `meta` and changes nothing about how the policy is evaluated. An unrecognised value has exactly the same effect as a recognised-looking one: none. + +In particular, `enforcement` never affects the exit code of the `tirith` command. The exit code is determined solely by `final_result` and the `--fail-on-error` flag (see [Outcomes and exit codes](#outcomes-and-exit-codes)). If you need a policy to block a pipeline *when invoking the CLI directly*, gate on the exit code with `--fail-on-error`, not on this field. The same applies to `severity`, `tags`, and `remediation`: they exist so that tools consuming Tirith's JSON output can act on them, and the engine passes them through untouched. + +:::note Consumers do interpret it + +The field is not decorative — it is read by the layer above the engine. The +[GitHub Action](../tirith-usage/ci-integration.md) downgrades a failing policy to a warning when +`meta.enforcement` is one of `soft_mandatory`, `advisory`, `warn`, `warning`, `low` or +`approval_required`, and blocks on `hard_mandatory`, `mandatory`, `fail`, `error`, `high`, +`critical` or `blocking`. Matching is case-insensitive and ignores surrounding whitespace. + +An **unrecognised** value blocks, and logs a warning that it did so. That is deliberate: a policy +that is mislabelled or carries a typo must gate rather than slip through silently. + +So `enforcement` is meaningful when a consumer acts on it, and inert when you run `tirith` yourself. +::: + +## `evaluators[]` + +Each entry in the `evaluators` array is an object with these keys: + +| Key | Required | Type | Default | Behavior | +|---|---|---|---|---| +| `id` | yes | string | — | The name this check is referenced by in `eval_expression`. Missing `id` aborts the run (`ERROR`, exit `1`). | +| `provider_args` | yes | object | — | Arguments for the provider selected by `meta.required_provider`. Missing `provider_args` aborts the run (`ERROR`, exit `1`). | +| `condition` | yes | object | — | The comparison applied to every value the provider extracts. Missing `condition` aborts the run (`ERROR`, exit `1`). | +| `description` | no | string | none | Informational. Echoed into the result for this check (as `null` when absent). | + +`id` is substituted into `eval_expression` as a bare word, so it must look like an identifier: letters, digits, and underscores. Ids should be unique within a policy; if two evaluators share an id, both appear in the output but only the **last** one's outcome is substituted into `eval_expression`. + +### `evaluators[].provider_args` + +The contents are provider-specific; the one key every provider expects is `operation_type`, which selects the operation (for example `get_value` for `stackguardian/json`, or `attribute` for `stackguardian/terraform_plan`). See [providers](../tirith-providers/overview.md) for each provider's operations and arguments. + +A malformed `provider_args` — an unsupported `operation_type`, or a missing required argument — does not abort the run. The provider reports the mistake as an error on that check, the check fails regardless of `error_tolerance`, and the message tells you what was wrong (for example `operation_type: 'attrbute' is not supported (severity_value: 99)`). + +### `evaluators[].condition` + +| Key | Required | Type | Default | Behavior | +|---|---|---|---|---| +| `type` | yes | string | — | The condition (evaluator) name. An unknown or missing `type` does not abort the run: that check fails with `` `X` is not a supported evaluator ``. | +| `value` | yes in practice | any | `null` | The operand the extracted value is compared against. The expected type depends on `type` (a list for `ContainedIn`, a pattern string for `RegexMatch`, a number for `LessThan`, and so on). Omitting it compares against `null`; the outcome then depends on the condition type, so always set it explicitly. | +| `error_tolerance` | no | integer | `0` | The maximum provider-error severity this check tolerates. Errors at or below the tolerance mark the check as **skipped** instead of failed. See [Error tolerance](#error-tolerance-the-third-outcome). | + +The supported condition types are: + +`ContainedIn`, `Contains`, `Equals`, `GreaterThan`, `GreaterThanEqualTo`, `IsEmpty`, `IsNotEmpty`, `LessThan`, `LessThanEqualTo`, `NotContainedIn`, `NotContains`, `NotEquals`, `RegexMatch` + +Their exact semantics are documented in the [evaluator reference](../tirith-reference/evaluators.md). + +A provider may extract several values for one check (for example, one attribute per matching resource). The condition is applied to each value, and the check passes only if **every** value passes. + +## `eval_expression` + +A boolean expression that combines the per-check outcomes into the final verdict. Operands are evaluator `id`s; the operators are: + +- `&&` — and +- `||` — or +- `!` — not +- `(` `)` — grouping + +`&` and `|` are rejected with an explicit error (`Unsupported operator '&' in eval_expression. Use '&&' instead.`) and the run aborts with exit code `1`. + +Two behaviors worth knowing: + +- **An id that does not match any evaluator is silently dropped from the expression**, and the run continues. The result carries an informational note in its `errors` array (`The following evaluator ids are not defined and have been removed: ...`), but this is not a failure: a policy whose expression is `real_check && typo_id` passes if `real_check` passes. Check the `errors` array (or the `Errors:` block in the printed output) when authoring. +- **Skipped checks are removed from the expression** before it is evaluated, rather than being treated as false. `a && b` with `b` skipped evaluates as just `a`. If every id in the expression is removed — all checks skipped — the final verdict is neither pass nor fail; see the next section. + +## Outcomes and exit codes + +Every check has one of three outcomes, reported in the `passed` field of its result: + +| `passed` | Meaning | +|---|---| +| `true` | Every value the provider extracted satisfied the condition. | +| `false` | At least one value failed the condition, the provider found no values at all (`Could not find input value`), or a provider error exceeded `error_tolerance`. | +| `null` | Skipped: the provider reported an error whose severity is within `error_tolerance`. | + +The final verdict, `final_result`, is also tri-state: `true` when the expression evaluates true, `false` when it evaluates false, and `null` when every check it references was skipped. + +The CLI exit code depends on `final_result` and the `--fail-on-error` flag: + +| Situation | `final_result` | Exit (default) | Exit (`--fail-on-error`) | +|---|---|---|---| +| Policy passed | `true` | 0 | 0 | +| Policy failed | `false` | 0 | 3 | +| All checks skipped | `null` | 0 | 1 | +| Unresolved variable | absent | 0 | 1 | +| Policy file malformed (missing `meta`, `evaluators`, `eval_expression`, `id`, `provider_args`, `condition`; `&` instead of `&&` in the expression, and likewise for the or operator) | — | 1 | 1 | + +Without `--fail-on-error` the exit code is `0` whether the policy passed or failed — the verdict is only in the output. A run where every check was skipped is deliberately treated as an error under `--fail-on-error`, not a pass: it verified nothing. See the [CLI reference](../tirith-usage/cli-reference.md) for the flag. + +## Error tolerance: the third outcome + +`error_tolerance` exists so a policy can tolerate *missing data* without tolerating *violations*. When a provider cannot extract a value, it reports an error with a numeric severity instead of a value. For each such error on a check: + +- severity **>** `error_tolerance` → the check **fails**, with the provider's message. +- severity **≤** `error_tolerance` → that result is **skipped** (`passed: null`), with the provider's message. + +The default tolerance is `0`. The severities the bundled providers use: + +| Severity | Used by | Meaning | +|---|---|---| +| 0 | `terraform_plan` | No resource changes in the plan at all, or a matched resource has no planned attributes (for example, a resource being destroyed). Because the comparison is *strictly greater than*, severity-0 errors are skipped even at the default tolerance of 0. | +| 1 | `terraform_plan` | The resource type was not found in the plan. | +| 2 | `terraform_plan`, `json` | The attribute (`terraform_plan`) or `key_path` (`json`) was not found. | +| 99 | `terraform_plan` | The policy itself is malformed (unsupported `operation_type`, missing required argument). Do not set a tolerance this high: it would mask broken policies. | + +So `"error_tolerance": 2` is the common setting for "skip this check when the key or attribute is absent", and `"error_tolerance": 1` for "skip when the resource type does not appear in the plan". + +Two situations are never tolerated, regardless of the setting: + +- The provider found **no values at all** for the check (`Could not find input value`) — this fails. +- The provider reported an error **without a severity**, which the engine treats as a malformed provider call — this fails. + +A skipped check interacts with the final verdict as described above: it is removed from `eval_expression`, and if nothing is left, `final_result` is `null` — reported as `= Skipped final evaluator`, exit `0` by default and exit `1` under `--fail-on-error`. + +## Variables + +Any **string** value in the policy can be replaced by a variable reference: + +```json +"condition": { + "type": "LessThanEqualTo", + "value": "{{ var.max_monthly_cost }}" +} +``` + +The rules, exactly as implemented: + +- The syntax is `{{ var.NAME }}`. The `var.` prefix is mandatory; `{{ NAME }}` is not a variable reference and is left untouched. +- The reference must start the string, and the whole string is replaced by the variable's value — which keeps the variable's JSON type. A number stays a number, a list stays a list. Variables cannot be interpolated into the middle of a longer string. +- Substitution is applied to: string values directly under `meta`, each evaluator's `id`, string values directly under `provider_args` and `condition`, and `eval_expression`. It does **not** recurse into nested objects or arrays inside those keys. +- `NAME` may be a dotted path (`{{ var.limits.cost }}`), looked up inside the variable document. + +Variables come from two CLI sources, applied in this order (later wins): + +1. `-var-path vars.json` — a JSON object per file; the flag may be repeated, and files are merged left to right, so a later file overrides an earlier one key by key. +2. `-var NAME=VALUE` — `VALUE` is parsed as JSON (`-var 'max_monthly_cost=300'`, `-var 'env="prod"'`); the flag may be repeated. Inline variables override variable files. An inline variable that is not of the form `NAME=` is ignored with a logged error — it does not define the variable. + +If a referenced variable is not defined by any source, the policy is **not evaluated at all**: the result contains only `{"errors": ["Variables not found: NAME"]}`, there is no verdict, and the CLI exits `0` by default and `1` under `--fail-on-error`. + +## Result document + +With `--json`, the CLI prints a single JSON object: + +| Key | Type | Content | +|---|---|---| +| `meta` | object | `version` and `required_provider` (always present, `null`/`"core"` when defaulted), plus whichever of `id`, `name`, `description`, `severity`, `enforcement`, `tags`, `remediation` the policy declared, copied verbatim. | +| `final_result` | `true` / `false` / `null` | The final verdict. | +| `evaluators` | array | One entry per check: `id`, `description`, tri-state `passed`, and `result` — the per-value messages, each with its own tri-state `passed`. | +| `errors` | array of strings | Informational notes from evaluating `eval_expression` (undefined ids that were removed, disallowed symbols). Empty on a clean run — including a clean *failing* run. | +| `eval_expression` | string | The expression that was evaluated, after variable substitution. | + +For complete, runnable policies with their inputs and verdicts, see the [Policy Cookbook](./tirith-policy-cookbook.md). diff --git a/documentation/docs/tirith-policies/tirith-policy-variables.md b/documentation/docs/tirith-policies/tirith-policy-variables.md index 5b4d2539..c37229f8 100644 --- a/documentation/docs/tirith-policies/tirith-policy-variables.md +++ b/documentation/docs/tirith-policies/tirith-policy-variables.md @@ -9,11 +9,12 @@ site_name: Tirith slug: tirith-policy-variables/ --- -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; - Policy variables allow dynamic values in policy definitions. They can be referenced in conditions to make policies more flexible. +A variable is referenced as `{{ var.NAME }}`. The `var.` prefix is required — a placeholder written +without it, such as `{{ max_epoch }}`, is not recognised as a variable and is compared as the +literal string, so the check quietly measures the wrong thing instead of failing. + ```json title="variables.json" { "max_epoch": 1720415598 @@ -23,7 +24,7 @@ Policy variables allow dynamic values in policy definitions. They can be referen ```json title="policy.json" { "meta": { - "version": "v1" + "version": "v1", "required_provider": "stackguardian/json" }, "evaluators": [ @@ -35,10 +36,11 @@ Policy variables allow dynamic values in policy definitions. They can be referen }, "condition": { "type": "LessThan", - "value": "{{ max_epoch }}" + "value": "{{ var.max_epoch }}" } } - ] + ], + "eval_expression": "epoch_check" } ``` @@ -48,3 +50,18 @@ Example command: tirith -input-path -policy-path policy.json -var-path variables.json ``` +Against an input of `{"meta": {"epoch": 1720000000}}` this passes: + +``` +Check: epoch_check + PASSED + 1. PASSED: `1720000000` is less than `1720415598` +``` + +Supply variables inline with `-var` instead of, or in addition to, `-var-path`; an inline `-var` +wins when the same name is set in both. A variable that is referenced but never supplied is an +error, not an empty value: the run reports `Variables not found` and exits `1`. + +See [Policy reference](tirith-policy-reference.md) for where `{{ var.NAME }}` may appear, and +[CLI reference](../tirith-usage/cli-reference.md) for the flags. + diff --git a/documentation/docs/tirith-providers/infracost.md b/documentation/docs/tirith-providers/infracost.md new file mode 100644 index 00000000..a3b694fc --- /dev/null +++ b/documentation/docs/tirith-providers/infracost.md @@ -0,0 +1,92 @@ +--- +id: infracost-provider +title: Infracost Provider +sidebar_label: Infracost +description: Reference for the stackguardian/infracost provider - operation types, parameters, return shapes, and error behavior. +keywords: + - tirith +site_name: Tirith +slug: infracost-provider/ +--- + +``` +required_provider: stackguardian/infracost +``` + +Sums estimated costs from an Infracost cost breakdown, either for all resources or for a chosen set of resource types. + +## Input document + +The JSON produced by Infracost: + +```bash +infracost breakdown --path . --format json > infracost.json +tirith -policy-path policy.json -input-path infracost.json +``` + +The provider reads `projects[].breakdown.resources[]` and understands both the older per-resource keys (`totalMonthlyCost` / `totalHourlyCost`) and the newer ones (`monthlyCost` / `hourlyCost`). Resources whose cost field is missing or `null` contribute nothing to the sum. + +Note: only the **first** project in the `projects` array is summed. + +## Operation types + +| `operation_type` | Purpose | +|---|---| +| `total_monthly_cost` | Sum of estimated monthly costs | +| `total_hourly_cost` | Sum of estimated hourly costs | + +Both operations take the same parameters: + +| Parameter | Required | Description | +|---|---|---| +| `operation_type` | yes | `total_monthly_cost` or `total_hourly_cost`. | +| `resource_type` | yes | Which resources to sum. `"*"`, `["*"]`, or an empty value sums **all** resources. Otherwise, a list of Terraform resource type names (e.g. `["aws_eks_cluster", "aws_s3_bucket"]`); a resource is included when the type part of its name (everything before the first `.`) is in the list. | + +**Returns:** a single number — the sum of the selected resources' costs. If nothing matches, the sum is `0`. + +**On a miss / error:** all errors from this provider carry **no severity value**, so they always fail the check and `error_tolerance` cannot skip them: + +- `operation_type` or `resource_type` key missing from `provider_args` — error `'resource_type/operation_type not found in provider_args'`. +- An `operation_type` other than the two above — error naming the unknown value. +- Input without a `projects` key — error `'projects not found in input_data'`. +- A project without `breakdown.resources` — error `'breakdown/resources not found in one of the project'`. + +## Example + +Verified end-to-end against the test fixtures — the total monthly cost of the stack must stay at or below 30, and the selected resource types must be free: + +```json +{ + "meta": { + "version": "v1", + "required_provider": "stackguardian/infracost" + }, + "evaluators": [ + { + "id": "cost_check_1", + "provider_args": { + "operation_type": "total_monthly_cost", + "resource_type": ["*"] + }, + "condition": { + "type": "LessThanEqualTo", + "value": 30 + } + }, + { + "id": "cost_check_2", + "provider_args": { + "operation_type": "total_monthly_cost", + "resource_type": ["aws_eks_cluster", "aws_s3_bucket"] + }, + "condition": { + "type": "Equals", + "value": 0 + } + } + ], + "eval_expression": "cost_check_1 && cost_check_2" +} +``` + +Condition types are documented in the [evaluators reference](../tirith-reference/evaluators.md); CLI flags in the [CLI reference](../tirith-usage/cli-reference.md). diff --git a/documentation/docs/tirith-providers/json.md b/documentation/docs/tirith-providers/json.md new file mode 100644 index 00000000..033b7509 --- /dev/null +++ b/documentation/docs/tirith-providers/json.md @@ -0,0 +1,141 @@ +--- +id: json-provider +title: JSON Provider +sidebar_label: JSON +description: Reference for the stackguardian/json provider - the get_value operation, key path syntax, return shapes, and error behavior. +keywords: + - tirith +site_name: Tirith +slug: json-provider/ +--- + +``` +required_provider: stackguardian/json +``` + +Extracts values from any JSON or YAML document by key path. Use this provider when no specialized provider exists for your input format. + +## Input document + +Any JSON file, or any YAML file (`.yaml` / `.yml` extension). A YAML file with multiple documents (separated by `---`) is parsed into a **list** of documents; start the key path with `*.` to iterate over them. + +## Operation types + +| `operation_type` | Purpose | +|---|---| +| `get_value` | Get the value(s) at a key path | + +Any other `operation_type` produces an error **without** a severity value, which always fails the check. + +--- + +## `get_value` + +| Parameter | Required | Description | +|---|---|---| +| `key_path` | yes | Dot-separated path into the document. `*` as a path segment iterates over every element of a list or every value of a dict. | + +Path syntax, with examples of what each returns: + +| `key_path` | Input | Values produced | +|---|---|---| +| `a.b` | `{"a": {"b": 1}}` | `1` | +| `c` | `{"c": ["aa", "bb"]}` | `["aa", "bb"]` (the whole list, one value) | +| `nested_map` | `{"nested_map": {"e": {"f": "3"}}}` | `{"e": {"f": "3"}}` (the whole dict, one value) | +| `list_of_dict.*.key1` | `{"list_of_dict": [{"key1": "value1"}, {"key1": "value1"}]}` | `"value1"`, `"value1"` (one value per element) | +| `countries.*.capital` | `{"countries": {"US": {"capital": "Washington"}, "UK": {"capital": "London"}}}` | `"Washington"`, `"London"` (one per dict value) | +| `*.name` | `[{"name": "Alice"}, {"name": "Bob"}]` | `"Alice"`, `"Bob"` (leading `*` over a top-level list) | + +**Returns:** one result per value found at the path. Without `*`, that is a single value of whatever shape lives there (scalar, list, or dict). With `*`, one result per matched element — and the condition must pass for **every** one of them. + +**On a miss:** if the path matches nothing, the provider reports an error with **severity 2** (`` key_path: `...` is not found ``). With the default `error_tolerance` of 0 the check fails; with `error_tolerance: 2` it is skipped instead. See [error tolerance](../tirith-policies/tirith-policy-error-tolerance.md). + +## Examples + +Verified end-to-end against the test fixtures: + +```json +{ + "meta": { + "version": "v1", + "required_provider": "stackguardian/json" + }, + "evaluators": [ + { + "id": "check0", + "provider_args": { + "operation_type": "get_value", + "key_path": "z.b" + }, + "condition": { + "type": "LessThanEqualTo", + "value": 1, + "error_tolerance": 2 + } + }, + { + "id": "check1", + "provider_args": { + "operation_type": "get_value", + "key_path": "a.b" + }, + "condition": { + "type": "LessThanEqualTo", + "value": 1 + } + }, + { + "id": "check2", + "provider_args": { + "operation_type": "get_value", + "key_path": "c" + }, + "condition": { + "type": "Contains", + "value": "aa" + } + }, + { + "id": "check4", + "provider_args": { + "operation_type": "get_value", + "key_path": "list_of_dict.*.key1" + }, + "condition": { + "type": "Equals", + "value": "value1" + } + } + ], + "eval_expression": "check1 && check2 && check4" +} +``` + +(`check0` targets a path that does not exist; with `error_tolerance: 2` it is skipped and dropped from `eval_expression` instead of failing.) + +The provider also works on YAML — this policy checks an Ansible playbook (a YAML file whose top level is a list of plays, hence the leading `*.`): + +```json +{ + "meta": { + "version": "v1", + "required_provider": "stackguardian/json" + }, + "evaluators": [ + { + "id": "check0", + "provider_args": { + "operation_type": "get_value", + "key_path": "*.vars.region" + }, + "condition": { + "type": "Equals", + "value": "your_aws_region" + } + } + ], + "eval_expression": "check0" +} +``` + +Condition types are documented in the [evaluators reference](../tirith-reference/evaluators.md). diff --git a/documentation/docs/tirith-providers/kubernetes.md b/documentation/docs/tirith-providers/kubernetes.md new file mode 100644 index 00000000..739d32c9 --- /dev/null +++ b/documentation/docs/tirith-providers/kubernetes.md @@ -0,0 +1,84 @@ +--- +id: kubernetes-provider +title: Kubernetes Provider +sidebar_label: Kubernetes +description: Reference for the stackguardian/kubernetes provider - the attribute operation, parameters, return shapes, and error behavior. +keywords: + - tirith +site_name: Tirith +slug: kubernetes-provider/ +--- + +``` +required_provider: stackguardian/kubernetes +``` + +Extracts attribute values from Kubernetes manifests of a chosen `kind` (Pod, Deployment, Service, ...). + +## Input document + +A **list** of Kubernetes manifests. In practice this is a multi-document YAML file — for example the output of `helm template` or a concatenation of manifests separated by `---`: + +```bash +helm template my-release ./chart > manifests.yml +tirith -policy-path policy.json -input-path manifests.yml +``` + +Every document in the list must have a `kind` key. Note that a YAML file containing only a **single** document does not currently work with this provider — the input must parse to a list of manifests (two or more YAML documents, or a JSON array). + +## Operation types + +| `operation_type` | Purpose | +|---|---| +| `attribute` | Get the value at an attribute path from every manifest of a kind | + +Any other `operation_type` produces an error **without** a severity value, which always fails the check. + +--- + +## `attribute` + +| Parameter | Required | Description | +|---|---|---| +| `kubernetes_kind` | yes | The `kind` to match, e.g. `Pod`, `Deployment`. Exact match. Omitting it produces a severity 99 error. | +| `attribute_path` | yes | Dot-separated path into the manifest, e.g. `spec.containers.*.image`. `*` as a path segment iterates over every element of a list or every value of a dict. Omitting it (or passing an empty string) produces a severity 99 error. | + +**Returns:** one result per manifest whose `kind` matches: + +- If `attribute_path` contains **no** `*` — the single value at that path (scalar, list, or dict), or `null` when the path is absent from that manifest. +- If `attribute_path` contains `*` — a **list** with one entry per matched element; elements where the remainder of the path is absent appear as `null` in the list. The condition is applied to the list as a whole, which makes `Contains` / `NotContains` (checking for `null` entries) the natural conditions to pair with wildcard paths. + +**On a miss:** no manifest of the requested kind — severity 1 (`kind: ... is not found`). A present kind with an absent path is **not** an error; it produces `null` values as described above. + +## Example + +Verified end-to-end against the test fixtures — every container of every `Pod` must define a `livenessProbe`: + +```json +{ + "meta": { + "version": "v1", + "required_provider": "stackguardian/kubernetes" + }, + "evaluators": [ + { + "id": "kinds_have_null_liveness_probe", + "provider_args": { + "operation_type": "attribute", + "kubernetes_kind": "Pod", + "attribute_path": "spec.containers.*.livenessProbe" + }, + "condition": { + "type": "Contains", + "value": null, + "error_tolerance": 2 + } + } + ], + "eval_expression": "!kinds_have_null_liveness_probe" +} +``` + +How this works: for each `Pod`, the provider returns the list of every container's `livenessProbe` value, with `null` for containers that lack one. The `Contains: null` condition is true when at least one container is missing the probe, and the `eval_expression` negates it, so the policy passes only when every container defines a probe. + +Condition types are documented in the [evaluators reference](../tirith-reference/evaluators.md). diff --git a/documentation/docs/tirith-providers/overview.md b/documentation/docs/tirith-providers/overview.md new file mode 100644 index 00000000..ede6d7ca --- /dev/null +++ b/documentation/docs/tirith-providers/overview.md @@ -0,0 +1,90 @@ +--- +id: providers-overview +title: Providers Overview +sidebar_label: Overview +description: What a Tirith provider is, how required_provider selects one, how provider_args are passed, and the list of available providers. +keywords: + - tirith +site_name: Tirith +slug: providers-overview/ +--- + +A **provider** is the part of Tirith that knows how to read one specific kind of input document and extract values from it. The policy declares which provider to use; each evaluator in the policy then asks the provider for values (via `provider_args`), and the evaluator's `condition` is applied to every value the provider returns. + +## Selecting a provider + +The provider is selected once for the whole policy with `meta.required_provider`: + +```json +{ + "meta": { + "version": "v1", + "required_provider": "stackguardian/terraform_plan" + }, + "evaluators": [ ... ], + "eval_expression": "..." +} +``` + +The value must be one of the exact strings below. If the string does not match any known provider, every evaluator in the policy fails with an error. + +| `required_provider` | Summary | Expected input document | +|---|---|---| +| [`stackguardian/terraform_plan`](terraform-plan.md) | Inspects resource changes, actions, counts, dependencies, references, provider configuration, and the Terraform version in a Terraform plan. | Terraform plan in JSON form (`terraform show -json `) | +| [`stackguardian/infracost`](infracost.md) | Sums estimated monthly or hourly costs from an Infracost breakdown. | Infracost output (`infracost breakdown --format json`) | +| [`stackguardian/json`](json.md) | Extracts values from any JSON or YAML document by key path, with wildcard support. | Any JSON or YAML file | +| [`stackguardian/kubernetes`](kubernetes.md) | Extracts attribute values from Kubernetes manifests of a given `kind`. | A list of Kubernetes manifests (multi-document YAML, e.g. `helm template` output) | +| [`stackguardian/sg_workflow`](sg-workflow.md) | Reads attributes of a StackGuardian workflow definition. | StackGuardian workflow JSON | + +## How `provider_args` reaches the provider + +Each evaluator carries a `provider_args` object. Tirith hands that object to the selected provider **verbatim** — the provider decides which keys it understands. Every provider except `stackguardian/sg_workflow` dispatches on the `operation_type` key; the remaining keys are parameters of that operation. + +```json +{ + "id": "my_check", + "provider_args": { + "operation_type": "attribute", + "terraform_resource_type": "aws_s3_bucket", + "terraform_resource_attribute": "force_destroy" + }, + "condition": { + "type": "Equals", + "value": false + } +} +``` + +The provider returns a **list of results**. Each result is a value extracted from the input (a scalar, a list, or a dict, depending on the operation). The evaluator's `condition` is applied to each value independently, and the evaluator passes only if **every** value passes. If the provider returns nothing at all, the evaluator fails with the message `Could not find input value`. + +For the available condition types (`Equals`, `Contains`, `RegexMatch`, ...) see the [evaluators reference](../tirith-reference/evaluators.md). + +## How the input document is parsed + +The file given to [`-input-path`](../tirith-usage/cli-reference.md) is parsed by extension: + +- `.yaml` / `.yml` — parsed as YAML. A file with multiple documents (separated by `---`) becomes a **list** of documents; a file with a single document becomes that document directly. +- anything else — parsed as JSON. + +The parsed value is what the provider sees. + +## Errors, misses, and `error_tolerance` + +When a provider cannot find what an operation asked for, it reports an error instead of a value. There are two kinds: + +1. **Errors with a severity value.** Most "not found" situations carry a numeric severity. Whether the check fails or is skipped depends on the evaluator's `condition.error_tolerance` (default `0`): + - severity **greater than** `error_tolerance` — the check **fails**. + - severity **less than or equal to** `error_tolerance` — the check is **skipped** (its `passed` is `null`, and its id is dropped from `eval_expression`). + + The conventional severity values are: + + | Severity | Meaning | + |---|---| + | 0 | Nothing to inspect (e.g. no resource changes in the plan). Skipped even at the default tolerance. | + | 1 | The requested resource / kind / provider was not found. | + | 2 | The resource was found but the requested attribute / key path was not. | + | 99 | The `provider_args` themselves are invalid (unsupported operation, missing required parameter). Practically never tolerated. | + +2. **Errors without a severity value.** Some errors (an unsupported `operation_type` in the `json` and `kubernetes` providers, and all errors from the `infracost` and `sg_workflow` providers) carry no severity. These always **fail** the check, regardless of `error_tolerance`. + +Each provider page below lists exactly which situation produces which severity. See also [error tolerance](../tirith-policies/tirith-policy-error-tolerance.md). diff --git a/documentation/docs/tirith-providers/sg-workflow.md b/documentation/docs/tirith-providers/sg-workflow.md new file mode 100644 index 00000000..f6403d80 --- /dev/null +++ b/documentation/docs/tirith-providers/sg-workflow.md @@ -0,0 +1,117 @@ +--- +id: sg-workflow-provider +title: SG Workflow Provider +sidebar_label: SG Workflow +description: Reference for the stackguardian/sg_workflow provider - supported workflow attributes, return shapes, and error behavior. +keywords: + - tirith +site_name: Tirith +slug: sg-workflow-provider/ +--- + +``` +required_provider: stackguardian/sg_workflow +``` + +Reads attributes of a StackGuardian workflow definition. + +## Input document + +A StackGuardian workflow definition in JSON form — the object that contains keys such as `WfType`, `TerraformConfig`, `VCSConfig`, and `DeploymentPlatformConfig`. + +## Parameters + +This provider does not dispatch on `operation_type`. It reads exactly one key from `provider_args`: + +| Parameter | Required | Description | +|---|---|---| +| `workflow_attribute` | yes | The name of the workflow attribute to read (see the table below). | + +By convention policies also set `"operation_type": "attribute"` (the test fixtures do), but the provider does not read or validate that key. + +## Supported values for `workflow_attribute` + +The attribute name determines where in the workflow document the value is read from: + +| `workflow_attribute` | Read from | Typical shape | +|---|---|---| +| `integrationId` | `DeploymentPlatformConfig[].config.integrationId`, with the `/integrations/` prefix stripped from each id | list of strings | +| `Description` | top level | string | +| `DocVersion` | top level | string | +| `ResourceName` | top level | string | +| `ResourceType` | top level | string (e.g. `WORKFLOW`) | +| `Tags` | top level | list of strings | +| `WfType` | top level | string (e.g. `TERRAFORM`) | +| `approvalPreApply` | `TerraformConfig` | boolean | +| `driftCheck` | `TerraformConfig` | boolean | +| `managedTerraformState` | `TerraformConfig` | boolean | +| `terraformVersion` | `TerraformConfig` | string | +| `bucket_region` | `VCSConfig.iacInputData.data` | string | +| `s3_bucket_acl` | `VCSConfig.iacInputData.data` | string | +| `s3_bucket_block_public_acls` | `VCSConfig.iacInputData.data` | boolean | +| `s3_bucket_block_public_policy` | `VCSConfig.iacInputData.data` | boolean | +| `s3_bucket_force_destroy` | `VCSConfig.iacInputData.data` | boolean | +| `s3_bucket_ignore_public_acls` | `VCSConfig.iacInputData.data` | boolean | +| `s3_bucket_restrict_public_buckets` | `VCSConfig.iacInputData.data` | boolean | +| `iacTemplateId` | `VCSConfig.iacVCSConfig` | string | +| `useMarketplaceTemplate` | `VCSConfig.iacVCSConfig` | boolean | + +**Returns:** a single value with the shape shown above. `integrationId` returns a list — pair it with `Contains` (see the example) rather than `Equals`. + +**On a miss / error:** all errors from this provider carry **no severity value**, so they always fail the check and `error_tolerance` cannot skip them: + +- The attribute's containing key is absent from the workflow document (e.g. no `TerraformConfig` when asking for `driftCheck`) — error `' not found in input_data'`. +- `workflow_attribute` missing from `provider_args` — error `workflow_attribute not found in provider_args`. +- `workflow_attribute` present but empty — the provider returns nothing and the check fails with `Could not find input value`. +- A `workflow_attribute` name that is not in the table above is **not** an error: the provider returns an empty string `""`, which is then evaluated against the condition. Double-check spelling — a typo silently evaluates `""` instead of the intended value. + +## Example + +Verified end-to-end against the test fixtures: + +```json +{ + "meta": { + "version": "v1", + "required_provider": "stackguardian/sg_workflow" + }, + "evaluators": [ + { + "id": "wf_check_1", + "provider_args": { + "operation_type": "attribute", + "workflow_attribute": "useMarketplaceTemplate" + }, + "condition": { + "type": "Equals", + "value": true + } + }, + { + "id": "wf_check_2", + "provider_args": { + "operation_type": "attribute", + "workflow_attribute": "integrationId" + }, + "condition": { + "type": "Contains", + "value": "aws-qa" + } + }, + { + "id": "wf_check_3", + "provider_args": { + "operation_type": "attribute", + "workflow_attribute": "terraformVersion" + }, + "condition": { + "type": "RegexMatch", + "value": "^1\\." + } + } + ], + "eval_expression": "wf_check_1 && wf_check_2 && wf_check_3" +} +``` + +Condition types are documented in the [evaluators reference](../tirith-reference/evaluators.md). diff --git a/documentation/docs/tirith-providers/terraform-plan.md b/documentation/docs/tirith-providers/terraform-plan.md new file mode 100644 index 00000000..d214b87b --- /dev/null +++ b/documentation/docs/tirith-providers/terraform-plan.md @@ -0,0 +1,377 @@ +--- +id: terraform-plan-provider +title: Terraform Plan Provider +sidebar_label: Terraform Plan +description: Reference for the stackguardian/terraform_plan provider - operation types, parameters, return shapes, and error behavior. +keywords: + - tirith +site_name: Tirith +slug: terraform-plan-provider/ +--- + +``` +required_provider: stackguardian/terraform_plan +``` + +Inspects a Terraform plan: attribute values of changed resources, the actions applied to them, resource counts, explicit dependencies, references between resources, provider configuration, and the Terraform version. + +## Input document + +The JSON representation of a Terraform plan: + +```bash +terraform plan -out=plan.out +terraform show -json plan.out > plan.json +tirith -policy-path policy.json -input-path plan.json +``` + +Most operations read the `resource_changes` array of the plan. If the plan contains no `resource_changes` at all, every operation reports an error with severity 0 (`No Terraform resources changes are found`), which is skipped at the default `error_tolerance` of 0. + +## Operation types + +| `operation_type` | Purpose | +|---|---| +| `attribute` | Get an attribute's planned value for every instance of a resource type | +| `action` | Get the plan actions (`create`, `update`, `delete`, ...) for a resource type | +| `count` | Count the changed instances of a resource type | +| `direct_dependencies` | Get the resource types listed in a resource's `depends_on` | +| `direct_references` | Get or check references between resources | +| `terraform_version` | Get the Terraform version that produced the plan | +| `provider_config` | Get the configuration of a Terraform provider (version constraint or region) | + +Any other `operation_type` produces an error with severity 99, which fails the check. + +--- + +## `attribute` + +Returns the planned (`change.after`) value of an attribute for every instance of a resource type. + +| Parameter | Required | Description | +|---|---|---| +| `terraform_resource_type` | yes | Resource type to match (e.g. `aws_s3_bucket`), or `*` to match every type. | +| `terraform_resource_attribute` | yes | Attribute to read. A plain top-level key (`force_destroy`), a dotted path (`tags.costcenter`), or a path containing `.*.` to iterate over a list (`ebs_block_device.*.encrypted`). | +| `exclude_resource_types` | no (default `[]`) | List of resource types to skip. Only applied when `terraform_resource_type` is `*`. | + +**Returns:** one value per matching resource instance. With a `.*.` wildcard, one value per list element; list elements that lack the attribute contribute `null`, so they are still evaluated. The value is whatever the attribute holds in the plan — scalar, list, or dict. + +**On a miss:** + +- No resource of the requested type in `resource_changes` — severity 1 (`resource_type: '...' is not found`). +- Resource found, attribute absent — severity 2 (`attribute: '...' is not found`), reported per resource instance that lacks it. +- Resource found but its `change.after` is empty (e.g. a destroy-only change) — severity 0 (`No Terraform changes found for resource type: '...'`). + +Example (adapted from a test fixture; requires every resource in the plan to carry a non-empty `costcenter` tag): + +```json +{ + "meta": { + "version": "v1", + "required_provider": "stackguardian/terraform_plan" + }, + "evaluators": [ + { + "id": "every_resource_has_costcenter_tag", + "description": "All resources must have a 'costcenter' tag with a non-empty value", + "provider_args": { + "operation_type": "attribute", + "terraform_resource_type": "*", + "terraform_resource_attribute": "tags.costcenter", + "exclude_resource_types": ["aws_iam_role_policy_attachment"] + }, + "condition": { + "type": "IsNotEmpty", + "value": "", + "error_tolerance": 1 + } + } + ], + "eval_expression": "every_resource_has_costcenter_tag" +} +``` + +--- + +## `action` + +Returns the actions Terraform plans to take on every instance of a resource type. Actions come straight from `change.actions` in the plan: `create`, `update`, `delete`, `no-op`, `read` (a replacement appears as both `delete` and `create`). + +| Parameter | Required | Description | +|---|---|---| +| `terraform_resource_type` | yes | Resource type to match, or `*` for every type. | +| `exclude_resource_types` | no (default `[]`) | List of resource types to skip. Only applied when `terraform_resource_type` is `*`. | + +**Returns:** one string per action per matching resource instance (a resource with actions `["delete", "create"]` yields two values). + +**On a miss:** no resource of the requested type — severity 1. + +Example (adapted from a test fixture; fails when a virtual network would be deleted): + +```json +{ + "meta": { + "version": "v1", + "required_provider": "stackguardian/terraform_plan" + }, + "evaluators": [ + { + "id": "vnet_is_deleted", + "provider_args": { + "operation_type": "action", + "terraform_resource_type": "azurerm_virtual_network" + }, + "condition": { + "type": "ContainedIn", + "value": ["delete"], + "error_tolerance": 2 + } + } + ], + "eval_expression": "!vnet_is_deleted" +} +``` + +--- + +## `count` + +Counts the instances of a resource type in `resource_changes`. + +| Parameter | Required | Description | +|---|---|---| +| `terraform_resource_type` | yes | Resource type to count, or `*` for every type. | +| `exclude_resource_types` | no (default `[]`) | List of resource types to skip. Only applied when `terraform_resource_type` is `*`. | + +**Returns:** a single integer. A type with no instances returns `0` — this operation never produces a "not found" error. + +Example: + +```json +{ + "id": "at_most_ten_vpcs", + "provider_args": { + "operation_type": "count", + "terraform_resource_type": "aws_vpc" + }, + "condition": { + "type": "LessThanEqualTo", + "value": 10 + } +} +``` + +--- + +## `direct_dependencies` + +Returns, for each resource of a type, the resource types named in its explicit `depends_on`. Only resources declared in the **root module** of the configuration are inspected. + +| Parameter | Required | Description | +|---|---|---| +| `terraform_resource_type` | yes | Resource type to inspect. Omitting it produces a severity 99 error. | + +**Returns:** one list of resource-type strings per matching resource (only the type part of each `depends_on` entry, i.e. `aws_s3_bucket.example` becomes `aws_s3_bucket`). A resource without `depends_on` yields an empty list. + +**On a miss:** no resource of the requested type in the configuration — severity 1. + +Example (verified against a test fixture; requires every EC2 instance to declare an explicit dependency on an S3 bucket): + +```json +{ + "meta": { + "version": "v1", + "required_provider": "stackguardian/terraform_plan" + }, + "evaluators": [ + { + "id": "ec2_depends_on_s3", + "description": "Make sure that EC2 instances have explicit dependency on S3 bucket", + "provider_args": { + "operation_type": "direct_dependencies", + "terraform_resource_type": "aws_instance" + }, + "condition": { + "type": "Contains", + "value": "aws_s3_bucket", + "error_tolerance": 2 + } + } + ], + "eval_expression": "ec2_depends_on_s3" +} +``` + +--- + +## `direct_references` + +Inspects references between resources (a reference is created when one resource's argument uses another resource's attribute, e.g. `security_groups = [aws_security_group.sg.id]`). It has three modes, chosen by which parameters are present: + +| Parameter | Required | Description | +|---|---|---| +| `terraform_resource_type` | yes | The resource type under inspection. | +| `referenced_by` | no | A resource type that should point **at** `terraform_resource_type`. | +| `references_to` | no | A resource type that `terraform_resource_type` should point **to**. | + +`referenced_by` and `references_to` are mutually exclusive — supplying both produces a severity 99 error. + +### Plain mode (neither `referenced_by` nor `references_to`) + +For each resource of `terraform_resource_type` declared in the **root module**, returns the list of resource types it references in its expressions. + +**Returns:** one list of resource-type strings per matching resource. + +**On a miss:** type not found in the configuration — severity 1. Omitting `terraform_resource_type` — severity 99. + +Example (verified against a test fixture): + +```json +{ + "id": "aws_elbs_have_direct_references_to_security_group", + "provider_args": { + "operation_type": "direct_references", + "terraform_resource_type": "aws_elb" + }, + "condition": { + "type": "Contains", + "value": "aws_security_group", + "error_tolerance": 2 + } +} +``` + +### `referenced_by` mode + +Checks that instances of `terraform_resource_type` are referenced by resources of type `referenced_by`. Instances that are only being destroyed are ignored. Unlike the plain mode, references are searched through the whole configuration, including child modules. + +**Returns:** one boolean per instance of `terraform_resource_type` — `true` if some `referenced_by` resource references it, `false` otherwise. Use `"condition": {"type": "Equals", "value": true}` to require that all instances are referenced. + +**On a miss:** no (non-destroyed) instance of `terraform_resource_type` — severity 1. + +Example (from a test fixture; every S3 bucket must have an intelligent-tiering configuration attached): + +```json +{ + "meta": { + "required_provider": "stackguardian/terraform_plan", + "version": "v1" + }, + "evaluators": [ + { + "id": "s3HasLifeCycleIntelligentTiering", + "description": "Make sure all aws_s3_bucket are referenced by aws_s3_bucket_intelligent_tiering_configuration", + "provider_args": { + "operation_type": "direct_references", + "terraform_resource_type": "aws_s3_bucket", + "referenced_by": "aws_s3_bucket_intelligent_tiering_configuration" + }, + "condition": { + "type": "Equals", + "value": true, + "error_tolerance": 0 + } + } + ], + "eval_expression": "s3HasLifeCycleIntelligentTiering" +} +``` + +### `references_to` mode + +Checks that every instance of `terraform_resource_type` references at least one resource of type `references_to`. Instances that are only being destroyed are ignored. + +**Returns:** a **single** boolean — `true` only if all instances reference the target type. + +**On a miss:** no (non-destroyed) instance of `terraform_resource_type` — severity 1. + +Example (verified against a test fixture): + +```json +{ + "id": "elbRefsToSecGroup", + "description": "Make sure ELBs references to security groups", + "provider_args": { + "operation_type": "direct_references", + "terraform_resource_type": "aws_elb", + "references_to": "aws_security_group" + }, + "condition": { + "type": "Equals", + "value": true, + "error_tolerance": 0 + } +} +``` + +--- + +## `terraform_version` + +Returns the Terraform version string recorded in the plan. + +No parameters besides `operation_type`. + +**Returns:** a single string (e.g. `"1.4.5"`), or `null` if the plan has no `terraform_version` key. + +Example (verified end-to-end): + +```json +{ + "id": "terraform_version_check", + "provider_args": { + "operation_type": "terraform_version" + }, + "condition": { + "type": "RegexMatch", + "value": "^1\\." + } +} +``` + +--- + +## `provider_config` + +Reads the configuration of a Terraform provider from `configuration.provider_config` in the plan. + +| Parameter | Required | Description | +|---|---|---| +| `terraform_provider_full_name` | yes | The provider's full registry name, e.g. `registry.terraform.io/hashicorp/aws`. Omitting it produces a severity 99 error. | +| `attribute` | yes | What to read. Must be `version_constraint` or `region` — anything else produces a severity 99 error. | + +**Returns:** one string per provider entry whose `full_name` matches: the version constraint (e.g. `">= 3.11.0, < 4.0.0"`) or the region. The region is only found when it is written as a constant in the configuration; a region supplied through a variable is reported as not found (severity 2). + +**On a miss:** + +- Matching provider found but the attribute is absent — severity 2 (`` `region` is not found in the provider_config ``). +- No provider with that `full_name` — severity 1. + +Example (verified end-to-end): + +```json +{ + "id": "aws_region_check", + "provider_args": { + "operation_type": "provider_config", + "terraform_provider_full_name": "registry.terraform.io/hashicorp/aws", + "attribute": "region" + }, + "condition": { + "type": "ContainedIn", + "value": ["eu-central-1", "eu-west-1"] + } +} +``` + +--- + +## Error severities used by this provider + +| Severity | Situation | +|---|---| +| 0 | No `resource_changes` in the plan, or the matched resource has no planned values (destroy-only change). | +| 1 | Resource type / provider name not found. | +| 2 | Attribute not found on a matched resource or provider config. | +| 99 | Invalid `provider_args` (unsupported operation or attribute, missing required parameter, both `referenced_by` and `references_to` given). | + +Whether a severity fails or skips the check depends on `condition.error_tolerance` — see the [providers overview](overview.md) and [error tolerance](../tirith-policies/tirith-policy-error-tolerance.md). Condition types are documented in the [evaluators reference](../tirith-reference/evaluators.md). diff --git a/documentation/docs/tirith-reference/eval-expressions.md b/documentation/docs/tirith-reference/eval-expressions.md new file mode 100644 index 00000000..970376a1 --- /dev/null +++ b/documentation/docs/tirith-reference/eval-expressions.md @@ -0,0 +1,118 @@ +--- +id: eval-expressions +title: Evaluation Expressions +sidebar_label: Eval Expressions +description: Reference for eval_expression, the boolean expression that combines evaluator results into a policy's final verdict. +keywords: + - tirith +site_name: Tirith +slug: eval-expressions/ +--- + +A policy's top-level `eval_expression` is a boolean expression over the `id`s of its evaluators. After every evaluator has produced its verdict, Tirith substitutes those verdicts into the expression and evaluates it; the outcome becomes `final_result` in the output. + +```json +{ + "meta": { "version": "v1", "required_provider": "stackguardian/json" }, + "evaluators": [ + { "id": "check_region", "provider_args": { "...": "..." }, "condition": { "...": "..." } }, + { "id": "check_tags", "provider_args": { "...": "..." }, "condition": { "...": "..." } }, + { "id": "check_budget", "provider_args": { "...": "..." }, "condition": { "...": "..." } } + ], + "eval_expression": "(check_region || check_tags) && check_budget" +} +``` + +## Referencing evaluators + +Evaluators are referenced by their `id`, written bare (no quotes, no prefix). Substitution matches ids as whole words, so one id being a prefix of another (`check` and `check_2`) is not a problem. + +Use only letters, digits, and underscores in ids that appear in the expression. An id with other characters (such as `-`) still works *if it is defined*, because it is replaced by its verdict before the expression is parsed — but if such an id is missing from the policy, the leftover text cannot be parsed as an expression and the whole run aborts (see [Unparseable expressions](#unparseable-expressions)). + +Each id stands for the tri-state verdict of its evaluator: + +- `true` — every value it checked passed, +- `false` — at least one value failed, +- *skipped* — the evaluator did not actually check anything (all of its provider errors were within `error_tolerance`). + +## Operators + +| Operator | Meaning | Example | +| --- | --- | --- | +| `&&` | logical AND | `check_a && check_b` | +| `\|\|` | logical OR | `check_a \|\| check_b` | +| `!` | logical NOT | `!check_a` | +| `( )` | grouping | `(check_a \|\| check_b) && check_c` | + +Whitespace is ignored. There are no comparison operators, literals, or function calls — only ids, the three operators above, and parentheses. + +**Precedence**, from tightest to loosest: `!`, then `&&`, then `||`. Both of these hold (verified against the implementation): + +- `a || b && c` means `a || (b && c)` — with `a` true and `b`, `c` false, the expression is true. +- `!a || b` means `(!a) || b` — with `a` and `b` both true, the expression is true. + +Use parentheses whenever the intent is not obvious. + +**Single `&` and `|` are rejected.** They are not silently treated as `&&`/`||`; the run aborts with an explicit error and exit code 1: + +``` +Unsupported operator '&' in eval_expression. Use '&&' instead. +``` + +## Skipped evaluators + +An evaluator whose verdict is *skipped* (`passed: null` in the output) is **removed from the expression** before evaluation, together with any `!` that applied to it, rather than being treated as false: + +- `skipped && other` reduces to `other`; +- `!skipped && other` also reduces to `other`; +- if *everything* in the expression is removed, `final_result` is `null` — see below. + +This is deliberate: treating a skipped check as `false` would fail policies through `!`-negations, and treating it as `true` would pass checks that never ran. + +## Missing evaluator ids + +An id used in the expression but not defined by any evaluator does **not** abort the run. It is removed from the expression the same way a skipped evaluator is, the rest of the expression is evaluated normally, and a note is appended to the top-level `errors` array of the output: + +``` +The following evaluator ids are not defined and have been removed: ghost_check +``` + +`errors` is informational: it does not affect `final_result` or the exit code. A policy whose expression is `real_check && ghost_check` passes with `final_result: true` and exit code 0 when `real_check` passes. Watch the `errors` array — a typo in an id silently weakens the policy. + +The reverse — an evaluator defined but never mentioned in the expression — still runs and appears in the output, but its verdict does not influence `final_result`. + +## Unparseable expressions + +If the expression cannot be parsed at all — a syntax error such as `check1 &&`, an empty string, a single `&`/`|`, or leftover text from a missing id that is not a valid identifier — the evaluation **aborts**: no result document is produced (with `--json`, the output is `{}`), and the process exits with code **1** regardless of `--fail-on-error`. This is the "tool error" exit code, distinct from a policy failure. + +As a safety measure the expression is evaluated with no access to builtins, and any symbol that survives id substitution is rejected (`The following symbols are not allowed: ...`) with `final_result: false`; the expression language cannot call functions or reach interpreter internals. + +## From expression result to exit code + +`final_result` is tri-state, and with `--fail-on-error` it maps to the exit code: + +| `final_result` | Meaning | Exit code with `--fail-on-error` | +| --- | --- | --- | +| `true` | every check that ran passed the expression | 0 | +| `false` | the expression evaluated to false | 3 | +| `null` | nothing was left to evaluate — every evaluator referenced in the expression was skipped or undefined | 1 | +| *(absent)* | the run aborted before a verdict (unparseable expression, undefined policy variables) | 1 | + +Without `--fail-on-error`, the exit code is 0 in all of these cases except an aborted run, which still exits 1; the verdict is only in the output. + +`null` is not a pass: a policy whose every check was skipped checked precisely nothing. + +## Worked example + +```json +"eval_expression": "!deprecated_api_used && (region_allowed || region_exempted)" +``` + +| `deprecated_api_used` | `region_allowed` | `region_exempted` | `final_result` | +| --- | --- | --- | --- | +| `false` | `true` | `false` | `true` | +| `true` | `true` | `false` | `false` | +| *skipped* | `true` | `false` | `true` — reduces to `(region_allowed \|\| region_exempted)` | +| *skipped* | *skipped* | *skipped* | `null` — exit 1 under `--fail-on-error` | + +For what makes an individual evaluator pass, fail, or get skipped, see [Evaluators and Conditions](./evaluators.md). diff --git a/documentation/docs/tirith-reference/evaluators.md b/documentation/docs/tirith-reference/evaluators.md new file mode 100644 index 00000000..c71a9240 --- /dev/null +++ b/documentation/docs/tirith-reference/evaluators.md @@ -0,0 +1,362 @@ +--- +id: evaluators +title: Evaluators and Conditions +sidebar_label: Evaluators +description: Complete reference for all Tirith condition types, their parameters, type handling, and pass/fail semantics. +keywords: + - tirith +site_name: Tirith +slug: evaluators/ +--- + +Every evaluator in a Tirith policy applies a **condition** to one or more values extracted by a provider. This page is the complete reference for all 13 condition types, including exactly how each one treats strings, numbers, lists, dictionaries, and `null`. + +## Anatomy of a condition + +An evaluator block looks like this: + +```json +{ + "id": "region_check", + "provider_args": { + "operation_type": "get_value", + "key_path": "region" + }, + "condition": { + "type": "Equals", + "value": "eu-central-1" + } +} +``` + +The `condition` object accepts three keys: + +| Key | Required | Meaning | +| --- | --- | --- | +| `type` | yes | One of the 13 evaluator names listed below. The name is case-sensitive. | +| `value` | yes, except for `IsEmpty` and `IsNotEmpty`, which ignore it | The value the extracted input is compared against. Any JSON type is accepted; each evaluator defines which types it supports. | +| `error_tolerance` | no (default `0`) | The maximum provider error severity that is *skipped* instead of failing the evaluator. See [Error Tolerance](../tirith-policies/tirith-policy-error-tolerance.md). | + +Throughout this page: + +- **input value** means a value the provider extracted from the input document (`evaluator_input` in the code), +- **condition value** means `condition.value` from the policy (`evaluator_data` in the code). + +A provider can return *several* input values for one evaluator (for example, a wildcard `key_path` such as `items.*`). The condition is applied to **each value independently, and the evaluator passes only if every value passes**. If the provider returns no values at all, the evaluator fails with the message `Could not find input value`. + +Each evaluator therefore ends in one of three states: + +- `passed: true` — every extracted value satisfied the condition, +- `passed: false` — at least one value did not (or an unrecoverable provider error occurred), +- `passed: null` — the evaluation was *skipped*: every provider error was within `error_tolerance` and no value was actually checked. + +## Failures versus errors + +This distinction matters for exit codes, so it is worth stating precisely: + +- **Evaluators never abort the run.** Every condition type catches internal exceptions. A type mismatch — comparing a string with a number, matching a regex against `null`, searching inside a boolean — produces `passed: false` with an explanatory message. It is reported and gated exactly like a genuine policy violation: with `--fail-on-error`, `final_result: false` exits with code **3**. +- Exit code **1** (a tool error rather than a verdict) is reserved for problems outside the evaluators: an unreadable policy or input file, undefined policy variables, an `eval_expression` that cannot be parsed (see [Evaluation Expressions](./eval-expressions.md)), or a run in which every evaluator was skipped (`final_result: null`). +- A misconfigured evaluator — an unsupported `condition.type` or an unsupported provider `operation_type` — is surfaced as an ordinary failed evaluator (`passed: false`) with an explanatory message, so under `--fail-on-error` it exits **3**, not 1. + +Without `--fail-on-error`, the process exits **0** regardless of the verdict; the verdict is only in the output. + +## Quick reference + +| `condition.type` | Passes when | `condition.value` | On a type mismatch | +| --- | --- | --- | --- | +| [`Equals`](#equals) | input value equals the condition value | any JSON | returns false (values of different types are simply not equal) | +| [`NotEquals`](#notequals) | input value differs from the condition value | any JSON | returns true (different types are not equal) | +| [`GreaterThan`](#comparisons-greaterthan-greaterthanequalto-lessthan-lessthanequalto) | `input > condition value` | number, string, or list (same type as input) | returns false, message carries the comparison error | +| [`GreaterThanEqualTo`](#comparisons-greaterthan-greaterthanequalto-lessthan-lessthanequalto) | `input >= condition value` | same | same | +| [`LessThan`](#comparisons-greaterthan-greaterthanequalto-lessthan-lessthanequalto) | `input < condition value` | same | same | +| [`LessThanEqualTo`](#comparisons-greaterthan-greaterthanequalto-lessthan-lessthanequalto) | `input <= condition value` | same | same | +| [`IsEmpty`](#isempty) | input is `null`, `""`, `[]`, or `{}` | ignored | returns false for numbers and booleans (never an error) | +| [`IsNotEmpty`](#isnotempty) | input is a **non-empty string, list, or dictionary** | ignored | returns false for numbers, booleans, and `null` | +| [`RegexMatch`](#regexmatch) | the pattern is found in the input | string (regular expression) | returns false for non-string/list/dict input; invalid pattern returns false with the regex error message | +| [`ContainedIn`](#containedin) | the input value occurs inside the condition value | string, list, or dictionary | returns false with an "unsupported data type" message | +| [`NotContainedIn`](#notcontainedin) | the input value does **not** occur inside the condition value | string, list, or dictionary | returns false (not true) with an "unsupported data type" message | +| [`Contains`](#contains) | the condition value occurs inside the input value | any JSON (input must be string, list, or dictionary) | returns false with an "unsupported data type" message | +| [`NotContains`](#notcontains) | the condition value does **not** occur inside the input value | any JSON (input must be string, list, or dictionary) | returns false (not true) with an "unsupported data type" message | + +Note the last two rows of each pair: **the `Not*` variants are not simple negations.** When the data has a type the evaluator does not support, *both* the positive and the negative form fail. If a value may be absent or of an unexpected type, test that explicitly (for example with `IsNotEmpty`) instead of relying on a `Not*` condition to pass. + +--- + +## Equals + +Passes when the input value equals the condition value. + +- Comparison is by value, with one normalization: **lists of scalars are sorted before comparing**, recursively, including lists nested inside dictionaries. `[1, 2]` equals `[2, 1]`, and `{"a": [2, 1]}` equals `{"a": [1, 2]}`. A list that mixes types (for example `[1, "a"]`) cannot be sorted and is compared in its original order. +- Numbers compare numerically: `1` equals `1.0`. +- Booleans compare as the numbers 1 and 0: `true` equals `1` and `false` equals `0`. +- Strings never equal numbers: `"1"` is **not** equal to `1`. +- `null` equals `null`. +- Dictionaries compare by keys and values; key order never matters. + +A type mismatch is not an error; the values are simply unequal and the check fails. + +```json +"condition": { "type": "Equals", "value": ["b", "a"] } +``` + +| Input value | Result | +| --- | --- | +| `["a", "b"]` | passes (list order ignored) | +| `["a", "b", "c"]` | fails | +| `"a,b"` | fails | + +## NotEquals + +The exact negation of [`Equals`](#equals), using the same normalization. It passes whenever `Equals` would fail, including on type mismatches: `"1"` NotEquals `1` passes. + +```json +"condition": { "type": "NotEquals", "value": "0.0.0.0/0" } +``` + +An input value of `"10.0.0.0/16"` passes; `"0.0.0.0/0"` fails. + +## Comparisons: GreaterThan, GreaterThanEqualTo, LessThan, LessThanEqualTo + +Each passes when `input value condition value` holds: + +| Type | Operator | +| --- | --- | +| `GreaterThan` | `>` | +| `GreaterThanEqualTo` | `>=` | +| `LessThan` | `<` | +| `LessThanEqualTo` | `<=` | + +Supported operand combinations (both sides must be of a comparable type): + +- **numbers** with numbers — the usual numeric comparison; integers and floats mix freely (`1 <= 1.5`). +- **booleans** with numbers — booleans act as 1 and 0 (`true >= 0` passes). +- **strings** with strings — lexicographic, case-sensitive character-by-character comparison (`"b" > "a"` passes). Note this is *not* numeric: `"10" < "9"`. +- **lists** with lists — element-by-element lexicographic comparison (`[1, 3] > [1, 2]` passes). + +Any other combination — a string against a number, `null` against anything — **returns false**, with the underlying comparison error as the message, for example: + +``` +'>' not supported between instances of 'str' and 'int' +``` + +This is a failed check (exit 3 under `--fail-on-error`), not a tool error. In particular, an input value of `null` can never pass a comparison. + +```json +"condition": { "type": "LessThanEqualTo", "value": 100 } +``` + +| Input value | Result | +| --- | --- | +| `42` | passes | +| `100` | passes | +| `"42"` | fails — `'<=' not supported between instances of 'str' and 'int'` | +| `null` | fails | + +## IsEmpty + +Passes when the input value is `null`, an empty string `""`, an empty list `[]`, or an empty dictionary `{}`. `condition.value` is ignored and may be omitted. + +Everything else is "not empty" — including `0` and `false`, which fail this check. + +```json +"condition": { "type": "IsEmpty" } +``` + +| Input value | Result | +| --- | --- | +| `null` | passes | +| `""`, `[]`, `{}` | passes | +| `0` | fails | +| `false` | fails | +| `"x"` | fails | + +## IsNotEmpty + +Passes **only** when the input value is a non-empty string, a non-empty list, or a non-empty dictionary. `condition.value` is ignored and may be omitted. + +`IsNotEmpty` is **not** the negation of `IsEmpty`. Numbers and booleans are not strings, lists, or dictionaries, so they fail `IsNotEmpty` — even though they also fail `IsEmpty`. An input value of `5` fails both checks. + +```json +"condition": { "type": "IsNotEmpty" } +``` + +| Input value | Result | +| --- | --- | +| `"x"`, `[1]`, `{"a": 1}` | passes | +| `""`, `[]`, `{}`, `null` | fails | +| `5` | fails (a number is neither empty nor "not empty") | +| `true` | fails | + +## RegexMatch + +Passes when the regular expression in `condition.value` is found **anywhere** in the input value (search semantics, not full match). Anchor the pattern with `^` and `$` if you need it to match the whole string. Patterns use Python regular expression syntax and are case-sensitive. + +Input handling: + +- a **string** input is matched directly (multi-line strings included); +- a **list** or **dictionary** input is first converted to its Python string form and the pattern is matched against that text. Note this form uses single quotes — `["a"]` becomes `['a']`, and `{"a": 2}` becomes `{'a': 2}` — not JSON. +- **numbers, booleans, and `null` are never coerced**: the check returns false. An input value of `42` does not match the pattern `"4"`, and `true` does not match `"True"`. + +The pattern itself must be a string; a non-string `condition.value` returns false. + +An **invalid pattern** does not abort the run: the check returns false and the message carries the regex error, for example `unterminated character set at position 1`. Under `--fail-on-error` this exits 3, like any other failed check. + +```json +"condition": { "type": "RegexMatch", "value": "^us-(east|west)-[12]$" } +``` + +| Input value | Result | +| --- | --- | +| `"us-east-1"` | passes | +| `"eu-central-1"` | fails | +| `42` (against pattern `"4"`) | fails — numbers are not coerced | + +## ContainedIn + +Asks: **is the input value inside `condition.value`?** The condition value is the container. Which check runs depends on the types of both sides: + +| Input value | Condition value | Check | +| --- | --- | --- | +| string | string | substring: passes if the input occurs anywhere in the condition value (`"amp"` is contained in `"example"`) | +| scalar (string, number, boolean, `null`) | list | element membership: passes if the input equals one of the list's elements | +| list | list | **element** membership, not subset: passes only if the whole input list is one *element* of the condition list. `["a", "b"]` is **not** contained in `["a", "b", "c"]`; `["a"]` *is* contained in `[["a"], ["b"]]`. Lists of scalars are sorted on both sides first, so element order does not matter (`[2, 1]` is found in `[[1, 2], [3]]`) | +| dictionary | dictionary | subset: passes if **every** key of the input exists in the condition value with an equal value | +| scalar | dictionary | key membership: passes if the input is one of the dictionary's keys | +| anything else | number, boolean, or `null` — or a non-string input against a string | **unsupported**: returns false with the message `... is an unsupported data type for evaluating against value in 'condition.value'` | + +Two quirks to be aware of: + +- The common "is this value in the allowed list" use is the *scalar in list* row. If the provider hands you a **list** and you want to check that each element is allowed, extract the elements individually (for example with a `*` wildcard in `key_path`) rather than testing the list itself, which would be an element-membership test. +- In the string-substring and key-in-dictionary forms, a *failing* check reports the message `Not evaluated` (with `passed: false`). The verdict is correct; only the message is unhelpful. + +```json +"condition": { "type": "ContainedIn", "value": ["t3.micro", "t3.small"] } +``` + +| Input value | Result | +| --- | --- | +| `"t3.micro"` | passes | +| `"m5.large"` | fails | +| `["t3.micro"]` | fails — a list is checked as one element, and `["t3.micro"]` is not an element | + +## NotContainedIn + +Asks: **is the input value absent from `condition.value`?** Broadly the negation of [`ContainedIn`](#containedin), with the same type table — but with two deliberate differences: + +- **Dictionaries:** passes if **no** key of the input has an equal value in the condition value. Keys of the input that are absent from the condition value are ignored. This makes the pair asymmetric: with input `{"a": 1, "b": 2}` and condition value `{"a": 1}`, `ContainedIn` fails (key `b` is missing from the container) *and* `NotContainedIn` also fails (key `a` matches). Both directions can fail for the same pair. +- **Unsupported types are still failures, not passes.** If the condition value is a number, boolean, or `null`, `NotContainedIn` returns false with the same "unsupported data type" message that `ContainedIn` produces. A check like `NotContainedIn: null` can never pass. + +```json +"condition": { "type": "NotContainedIn", "value": ["0.0.0.0/0", "::/0"] } +``` + +| Input value | Result | +| --- | --- | +| `"10.0.0.0/16"` | passes | +| `"0.0.0.0/0"` | fails | + +## Contains + +The mirror image of [`ContainedIn`](#containedin): asks **does the input value contain `condition.value`?** Here the *input* is the container: + +| Input value | Condition value | Check | +| --- | --- | --- | +| string | string | substring: passes if the condition value occurs anywhere in the input (`"hello world"` contains `"world"`) | +| list | scalar | element membership | +| list | list | **element** membership, not subset: `["a", "b", "c"]` does not contain `["a", "b"]`, but `[["a"], "b"]` contains `["a"]`. Lists of scalars are sorted on both sides first | +| dictionary | dictionary | subset: passes if every key/value pair of the condition value exists in the input | +| dictionary | scalar | key membership: passes if the condition value is one of the input's keys | +| number, boolean, or `null` input | anything | **unsupported**: returns false with an "unsupported data type" message | + +An empty list or empty dictionary input contains nothing, so any search in it fails (except the degenerate `{}` contains `{}`, which passes). Unlike `ContainedIn`, failure messages here are always informative (`Failed to find ... inside ...`). + +The practical difference from `ContainedIn`: use `Contains` when the *extracted value* is the collection ("the tags attached to this resource must include X"); use `ContainedIn` when the *policy* holds the collection ("this value must be one of the allowed options"). + +```json +"condition": { "type": "Contains", "value": {"Environment": "production"} } +``` + +| Input value | Result | +| --- | --- | +| `{"Environment": "production", "Team": "core"}` | passes | +| `{"Environment": "staging", "Team": "core"}` | fails | +| `null` | fails — unsupported input type | + +## NotContains + +Asks: **does the input value *not* contain `condition.value`?** Broadly the negation of [`Contains`](#contains), with the same two departures the other `Not*` evaluator has: + +- **Dictionaries:** passes if **no** key/value pair of the condition value matches the input. Keys of the condition value that are absent from the input are ignored — `{"z": 1}` is "not contained" in `{"a": 1}` and the check passes. +- **Unsupported input types are failures, not passes.** If the input value is a number, boolean, or `null`, `NotContains` returns false — it does not treat "cannot contain anything" as "does not contain it". An absent (`null`) value therefore fails *both* `Contains` and `NotContains`. (The failure message in this case quotes the condition value rather than the input value.) + +```json +"condition": { "type": "NotContains", "value": "0.0.0.0/0" } +``` + +| Input value | Result | +| --- | --- | +| `["10.0.0.0/16", "192.168.0.0/24"]` | passes | +| `["10.0.0.0/16", "0.0.0.0/0"]` | fails | +| `null` | fails — unsupported input type | + +--- + +## Worked example + +Policy (`policy.json`), using the `stackguardian/json` provider: + +```json +{ + "meta": { + "version": "v1", + "required_provider": "stackguardian/json" + }, + "evaluators": [ + { + "id": "region_allowed", + "provider_args": { + "operation_type": "get_value", + "key_path": "region" + }, + "condition": { + "type": "ContainedIn", + "value": ["eu-central-1", "eu-west-1"] + } + }, + { + "id": "instances_are_small", + "provider_args": { + "operation_type": "get_value", + "key_path": "instances.*.count" + }, + "condition": { + "type": "LessThanEqualTo", + "value": 3 + } + } + ], + "eval_expression": "region_allowed && instances_are_small" +} +``` + +Input (`input.json`): + +```json +{ + "region": "eu-central-1", + "instances": [ + { "name": "web", "count": 2 }, + { "name": "worker", "count": 5 } + ] +} +``` + +Run: + +```bash +tirith -policy-path policy.json -input-path input.json +``` + +`region_allowed` passes. `instances_are_small` receives *two* input values from the wildcard (`2` and `5`); `2 <= 3` passes but `5 <= 3` fails, so the whole evaluator fails and `final_result` is `false`. With `--fail-on-error` the process exits with code 3. + +How the per-evaluator verdicts combine into `final_result` is defined by the policy's `eval_expression` — see [Evaluation Expressions](./eval-expressions.md). diff --git a/documentation/docs/tirith-usage/ci-integration.md b/documentation/docs/tirith-usage/ci-integration.md new file mode 100644 index 00000000..68cab83c --- /dev/null +++ b/documentation/docs/tirith-usage/ci-integration.md @@ -0,0 +1,146 @@ +--- +id: ci-integration +title: CI Integration +sidebar_label: CI Integration +description: Running Tirith in GitHub Actions via the action, and in GitLab CI or any container-based CI via the CLI directly. +keywords: + - tirith + - ci + - github actions + - gitlab +site_name: Tirith +slug: ci-integration/ +--- + +Tirith reads the plan your pipeline already produces — the output of +`terraform show -json tfplan` — checks it against your policies, and exits non-zero so a violating +change never reaches `apply`. The same policy files gate a GitHub Actions job, a GitLab job and a +laptop. + +Two ways to run it in CI: + +- **GitHub Actions** — use the + [StackGuardian/tirith-iac-governance-action](https://github.com/StackGuardian/tirith-iac-governance-action), + which wraps the CLI and adds the GitHub-specific reporting. +- **Everything else** — GitLab CI, or any CI that can run a container — invoke the CLI directly, + which is all the action does underneath. + +Either way, the job is gated by the [exit code](exit-codes.md): pass `--fail-on-error` (or the +action's `fail-on-error` input) and a failing policy fails the job. + +## GitHub Actions + +The action finds the plan, evaluates the policies, posts a sticky pull-request comment, creates a +check run and sets the job's exit code: + +```yaml +permissions: + contents: read + pull-requests: write # sticky comment + checks: write # check run + +steps: + - run: | + terraform plan -out=tfplan -input=false + terraform show -json tfplan > plan.json + + - uses: StackGuardian/tirith-iac-governance-action@v2 +``` + +With a `plan.json` in the working directory that is the whole integration — no `with:` block. The +action finds the document by convention (`plan.json` or `tfplan.json`) and evaluates the policy +files committed under `.tirith/policies`, on the runner, talking to nothing. Add +`with: { fail-on-error: true }` to make a failing policy fail the job. + +### Local mode and platform mode + +The action has two modes, chosen by whether credentials are present — there is no switch: + +- **Without credentials** (the default), policy files from your repository are evaluated on the + runner. Nothing is uploaded and no account is needed. +- **With credentials**, the action evaluates the policies your StackGuardian organization enforces + instead, by way of `tirith platform check` — see [Platform Check](platform-check.md): + +```yaml +env: + SG_API_TOKEN: ${{ secrets.SG_API_TOKEN }} + SG_ORG: ${{ vars.SG_ORG }} +``` + +Everything on the pull request is the same in both modes — the same comment, the same +`Tirith IaC Governance` check run, the same outputs and exit codes. + +### Commonly used inputs + +Every input is optional. The full list, with the matrix/monorepo guidance and the outputs, is in +the [action's own README](https://github.com/StackGuardian/tirith-iac-governance-action#readme). + +| Input | Default | | +|---|---|---| +| `policy-path` | `.tirith/policies` | Local mode only: a file, directory or glob of policy files | +| `input-path` | `plan.json` / `tfplan.json` | Document to evaluate, found by convention | +| `plan-file` | | Binary plan, rendered with `terraform show -json` in memory | +| `input-kind` | `terraform_plan` | `terraform_plan`, `terraform_state`, `kubernetes`, `json` | +| `fail-on-error` | `false` | Fail the job when a policy fails | +| `sg-region` | `eu` | `eu` or `us`; platform mode only | +| `source-dir` | `.` | Platform mode: the terraform source uploaded with the documents; `""` sends documents only | +| `timeout` | `1800` | Platform mode: seconds to wait for the run | + +The action's exit behaviour follows the shared contract: `fail-on-error` governs policy verdicts, +while a run that errored, an unreachable platform, or a job with no credentials *and* no policies +is always red — a check that gated nothing must not report green. + +### Outputs + +The action exposes `verdict` (`passed` | `warned` | `failed` | `errored` | `no-policies`), `mode` +(`platform` | `local`), the `passed` / `failed` / `warned` counts, the full result document as +`results` and `results-file`, and — in platform mode — `wfrun-id` and `wfrun-url` linking to the +run. + +## GitLab CI + +There is no GitLab-native equivalent of the action, so you invoke the CLI directly. Given an +earlier job that saved `plan.json` as an artifact: + +```yaml +policy: + image: python:3.12 + needs: [plan] + script: + - pip install "git+https://github.com/StackGuardian/tirith.git@1.0.5" + - tirith -policy-path .tirith/policies -input-path plan.json --fail-on-error +``` + +Tirith is **not on PyPI** — `pip install tirith` installs an unrelated project of the same name. +Install from git, and pin a tag rather than tracking the default branch so a CI job cannot change +behaviour underneath you. `1.0.5` is the newest tag; +`git ls-remote --tags https://github.com/StackGuardian/tirith.git` lists them. Python 3.8 or newer. + +To evaluate your organization's policies instead of the committed files, swap the last line for +`tirith platform check` and supply credentials as CI variables: + +```yaml +policy: + image: python:3.12 + needs: [plan] + variables: + SG_ORG: my-org # SG_API_TOKEN comes from a masked CI/CD variable + script: + - pip install "git+https://github.com/StackGuardian/tirith.git@1.0.5" + - tirith platform check --workflow-id my-repo --input-path plan.json --fail-on-error +``` + +See [Platform Check](platform-check.md) for what that uploads and what it masks first. + +## Any container-based CI + +Nothing above is GitLab-specific: any runner that can execute a container and produce a plan works +the same way. The recipe is always the same three steps — + +1. produce the input document (`terraform show -json tfplan > plan.json`); +2. `pip install "git+https://github.com/StackGuardian/tirith.git@1.0.5"`; +3. `tirith -policy-path -input-path plan.json --fail-on-error` + +— and gate the job on the exit code, which every CI system does by default for a non-zero exit. +Use `--json` to capture the result document for a later step, and see [Exit codes](exit-codes.md) +for telling a policy failure (`3`) apart from a tooling failure (`1`). diff --git a/documentation/docs/tirith-usage/cli-reference.md b/documentation/docs/tirith-usage/cli-reference.md new file mode 100644 index 00000000..b8a726c5 --- /dev/null +++ b/documentation/docs/tirith-usage/cli-reference.md @@ -0,0 +1,150 @@ +--- +id: cli-reference +title: CLI Reference +sidebar_label: CLI Reference +description: Every flag of the tirith command, what it prints, and how --json and --verbose change the output. +keywords: + - tirith + - cli +site_name: Tirith +slug: cli-reference/ +--- + +The base `tirith` command evaluates a policy file against an input document, locally, on your own +machine. Nothing is sent anywhere and no account is needed. + +``` +tirith -policy-path policy.json -input-path plan.json +``` + +Run with no arguments, `tirith` prints its help text and exits `0`. + +There is one subcommand, `tirith platform check`, which evaluates against the policies a +StackGuardian organization enforces instead of local files. It has its own flags and its own page: +[Platform Check](platform-check.md). + +## Flags + +Note the spelling: the path and variable options take a **single dash** (`-policy-path`, not +`--policy-path`), while the output and behaviour switches take two. + +| Flag | Argument | What it does | +|---|---|---| +| `-policy-path` | `PATH` | Path to the Tirith policy file. Required. | +| `-input-path` | `PATH` | Path to the document the policy is evaluated against. Required. | +| `-var-path` | `PATH` | Path to a JSON file of policy variables. Repeatable. | +| `-var` | `NAME=JSON` | One inline policy variable. Repeatable. | +| `--json` | | Print only the result document as JSON on stdout. | +| `--verbose` | | Show detailed (debug-level) logs from the run on stderr. | +| `--fail-on-error` | | Exit `3` when a policy fails, instead of `0`. Off by default. | +| `--version` | | Print the version and exit. | +| `-h`, `--help` | | Print the help text and exit. | + +### `-policy-path` + +The policy file to evaluate — a JSON document with `meta`, `evaluators` and an `eval_expression`. +See the [policy reference](../tirith-policies/tirith-policy-reference.md) for the schema, the +[evaluators reference](../tirith-reference/evaluators.md) for the available condition types, and the +[providers overview](../tirith-providers/overview.md) for what kinds of input each +`required_provider` reads. + +If the flag is missing, `tirith` prints an error to stderr and exits `1`. + +### `-input-path` + +The document to evaluate: a terraform plan in JSON form (`terraform show -json tfplan`), a +Kubernetes manifest, an Infracost breakdown, or any JSON document — whatever the policy's provider +expects. Files ending in `.yaml` or `.yml` are parsed as YAML; a multi-document YAML file is read +as a list of documents. Everything else is parsed as JSON. + +If the flag is missing, `tirith` prints an error to stderr and exits `1`. + +### `-var-path` and `-var` + +A policy can be parameterized with `{{ var.name }}` placeholders. These two flags supply the +values: + +``` +tirith -policy-path policy.json -input-path plan.json \ + -var-path common-vars.json -var 'max_cost=100' +``` + +- `-var-path` names a JSON file whose top-level keys are variable names. The flag may be repeated; + files are merged in order, and a later file overrides an earlier one for the same key. +- `-var` supplies a single variable inline as `name=value`, where `value` is parsed as JSON — so + `-var 'max_cost=100'` is a number, `-var 'region="eu-central-1"'` is a string, and + `-var 'allowed=["a","b"]'` is a list. Inline variables are applied after all files, so they + override them. A value that is not valid JSON is reported as an error and the variable is not + set. + +If the policy references a variable that none of these supplied, evaluation does not run at all: +the result carries only an `errors` entry (`Variables not found: ...`), and with +`--fail-on-error` the exit code is `1` — the tool could not evaluate, which is different from a +policy failing. + +### `--json` + +Prints the result document, and nothing else, to stdout — all logging is disabled, so the output +can be piped straight into `jq` or another program: + +``` +tirith -policy-path policy.json -input-path plan.json --json | jq .final_result +``` + +The document has this shape: + +```json +{ + "meta": { "version": "v1", "required_provider": "stackguardian/json" }, + "final_result": true, + "evaluators": [ + { + "id": "check1", + "passed": true, + "result": [ { "passed": true, "message": "1 is equal to 1", "meta": null } ], + "description": null + } + ], + "errors": [], + "eval_expression": "check1" +} +``` + +`final_result` is tri-state: `true` when every check that ran passed, `false` when a check ran and +failed, and `null` when every check was skipped (see +[error tolerance](../tirith-policies/tirith-policy-error-tolerance.md)) — the policy then evaluated +nothing. Each evaluator's `passed` is tri-state in the same way. If evaluation throws an +unexpected error under `--json`, the command prints an empty `{}` and exits `1`. + +The exit code does not change under `--json`; combine it with `--fail-on-error` to gate on the +verdict while still capturing the document. + +### `--verbose` + +Without it, the run prints the pretty-printed per-check results on stdout and only messages at +INFO level and above on stderr, formatted as `[LEVEL] message`. With `--verbose`, stderr carries +debug-level logs in a long format that includes the timestamp, process id and source location — +useful when a policy is not matching what you expect and you want to see each evaluator being +processed. + +`--verbose` has no effect together with `--json`, which disables logging entirely. + +### `--fail-on-error` + +By default the command exits `0` whenever it completed the evaluation, whether the policy passed +or failed — the verdict is in the output. `--fail-on-error` turns the exit code into a gate: `3` +when a policy failed, `1` when nothing could be evaluated, `0` only when every check that ran +passed. This is the flag that makes the command usable as a CI gate; the full contract is on the +[exit codes](exit-codes.md) page. + +### `--version` + +Prints the version number (for example `1.2.0`) and exits `0`. + +## Output streams + +- **stdout** carries the result: the pretty-printed report by default, or the JSON document under + `--json`. +- **stderr** carries logs and error messages. + +This split is deliberate so that redirecting stdout captures only the verdict. diff --git a/documentation/docs/tirith-usage/exit-codes.md b/documentation/docs/tirith-usage/exit-codes.md new file mode 100644 index 00000000..cac5ad99 --- /dev/null +++ b/documentation/docs/tirith-usage/exit-codes.md @@ -0,0 +1,93 @@ +--- +id: exit-codes +title: Exit Codes +sidebar_label: Exit Codes +description: The complete Tirith exit-code contract, and how to gate a CI job on it. +keywords: + - tirith + - exit codes + - ci +site_name: Tirith +slug: exit-codes/ +--- + +Tirith's exit codes are a contract shared by both surfaces — local evaluation (`tirith`) and +platform evaluation (`tirith platform check`) — so a caller scripting both only has to learn one +vocabulary. + +| Code | Meaning | +|---|---| +| `0` | Policies passed, or nothing was in scope to gate on | +| `1` | Tirith could not complete the evaluation — bad input, a policy it could not evaluate, unreachable API | +| `2` | Timed out waiting for a StackGuardian run (`tirith platform check` only; local evaluation never produces it) | +| `3` | A policy failed. Only with `--fail-on-error`, on either surface | +| `130` | Interrupted (Ctrl-C) | + +## `3` is deliberately not `1` + +`3` means a check ran and said no: your infrastructure violates a policy. `1` means Tirith could +not tell you either way — an unparseable policy file, an unresolved `{{ var.x }}` variable, an +unreachable API, or a policy whose every check was skipped. A job that treats every non-zero code +alike reports an outage as a policy violation, and cannot tell a working gate from a broken one. +Keeping the two codes distinct lets a pipeline page the platform team on `1` and the change author +on `3`. + +Both surfaces **fail closed**: anything that leaves the verdict unknown exits non-zero regardless +of `--fail-on-error`. That flag governs policy verdicts, not tool health — a run that produced no +verdict must never look like a pass. + +## Without `--fail-on-error` + +The local command exits `0` whether the policy passed or failed, with the verdict in the output. +That is how it has always behaved, and it is left alone so that upgrading Tirith cannot turn a +passing pipeline red; the gate is opt-in. `tirith platform check` behaves the same way: without +the flag a policy failure logs a message and still exits `0`, and the verdict is in +`--output-json`. + +Errors are different: a missing input file, an unparseable policy or an unresolved variable exits +`1` even without the flag. + +## What each local outcome produces + +Under `--fail-on-error`, the exit code is decided by the result's tri-state `final_result`: + +| `final_result` | Meaning | Exit | +|---|---|---| +| `true` | every check that ran passed | `0` | +| `false` | a check ran and failed | `3` | +| `null` | every check was skipped — the policy evaluated nothing | `1` | +| absent | the policy could not be evaluated at all (for example an unresolved variable) | `1` | + +`null` is not a pass. A policy whose every check was skipped — an +[`error_tolerance`](../tirith-policies/tirith-policy-error-tolerance.md) swallowing a provider that +found nothing — checked precisely nothing, and reporting that as green is exactly what the flag +exists to prevent. It is not a violation either, so it is `1` rather than `3`. + +**One limit worth stating plainly:** a *misconfigured* policy — an unsupported `condition.type`, +an unknown `required_provider` — comes back from the engine as an ordinary failed check with no +error attached, so it is indistinguishable from a real violation and exits `3`. It fails closed, +which is the safe direction, but it will point at your infrastructure when the fault is in the +policy. + +## Gating a CI job + +Most CI systems fail a job on any non-zero exit, so the minimal gate is one line: + +```sh +tirith -policy-path .tirith/policies -input-path plan.json --fail-on-error +``` + +To act differently on "policy failed" versus "Tirith broke", branch on the code: + +```sh +tirith -policy-path .tirith/policies -input-path plan.json --fail-on-error --json > result.json +code=$? +case "$code" in + 0) echo "policies passed" ;; + 3) echo "a policy failed — see result.json" ; exit 1 ;; + *) echo "Tirith could not evaluate (exit $code) — this is a tooling problem, not a verdict" ; exit "$code" ;; +esac +``` + +The same pattern works for `tirith platform check` unchanged — the codes mean the same things. +Complete CI examples are on the [CI integration](ci-integration.md) page. diff --git a/documentation/docs/tirith-usage/platform-check.md b/documentation/docs/tirith-usage/platform-check.md new file mode 100644 index 00000000..34b9cf69 --- /dev/null +++ b/documentation/docs/tirith-usage/platform-check.md @@ -0,0 +1,204 @@ +--- +id: platform-check +title: Platform Check +sidebar_label: Platform Check +description: The tirith platform check subcommand — every flag, what it uploads, what it masks on your machine first, and what it reports back. +keywords: + - tirith + - platform check + - stackguardian +site_name: Tirith +slug: platform-check/ +--- + +`tirith platform check` evaluates a terraform plan, state document or cost breakdown against the +policies your **StackGuardian organization** enforces, from any CI system or from a laptop — +instead of policy files committed to your repository. Policy then lives in one place rather than +being copied into every repository that needs gating. + +This is the one part of Tirith that talks to a network and needs an account. Plain +`tirith` — local evaluation — needs neither; see the [CLI reference](cli-reference.md). + +```sh +export SG_API_TOKEN=sgo_... # an organization token +export SG_ORG=my-org + +tirith platform check --workflow-id my-repo --input-path plan.json --fail-on-error +``` + +`--input-path` is optional when a `plan.json` or `tfplan.json` is in the working directory. + +On GitHub, prefer the +[GitHub Action](https://github.com/StackGuardian/tirith-iac-governance-action), which is a thin +wrapper around this command and adds the pull-request comment and check run — see +[CI integration](ci-integration.md). Use this command directly anywhere else: GitLab CI, a +Makefile, a local shell. + +## What it does + +1. **Masks the document on your machine**, before anything is uploaded (details below). +2. **Packs** the masked documents together with your terraform source into a `tar.gz`, excluding + `.git`, `.terraform`, `*.tfstate*` and anything matched by `.gitignore`. `--no-source` sends + documents only. An oversized source tree degrades to documents-only with a warning rather than + failing the check. +3. **Uploads** the archive to the workflow's artifact directory and creates a StackGuardian + workflow run. The workflow and its group are created on first use. +4. **Polls** the run until it finishes and prints the verdict — optionally also as JSON and + markdown files for a later CI step. + +## What it masks + +Masking happens client-side, on your machine, before anything leaves it. Masked values are +replaced with the sentinel `__SG_REDACTED__`. + +For a **terraform plan** (`--input-kind terraform_plan`, the default): + +- Every value terraform marked sensitive (`before_sensitive` / `after_sensitive`) is masked, in + `resource_changes`, `resource_drift` and `output_changes` alike. +- Root `variables` are dropped wholesale — the plan does not reliably mark which were declared + `sensitive`, so the only safe assumption is that all of them might be. +- `prior_state` is dropped, and terraform's own `planned_values` — which mirrors every value with + no sensitivity markers — is dropped and **rebuilt from the already-masked** `resource_changes`, + so tools that read that section still work without the leak. +- Credential-bearing literals in `configuration` (provider blocks, resource expressions, module + arguments, variable defaults) are scrubbed while the reference graph policies read is kept. +- Finally, any string terraform marked sensitive *somewhere* is masked *everywhere* in the + document — catching provider-computed mirrors such as `tags_all` that carry the same plaintext + without a marker of their own. + +For a **terraform state document** (`--input-kind terraform_state`, or `--state-path`): outputs +marked `sensitive` and every attribute named in an instance's `sensitive_attributes` are masked, +and the same everywhere-sweep is applied. Both shapes are handled — raw `terraform state pull` +output and `terraform show -json` output. + +Two limits, stated plainly: + +- **`json` and `kubernetes` documents are not masked** — there is no schema that says which fields + are secret. A document that looks like terraform state but is sent with the wrong `--input-kind` + triggers a warning, because that is the mistake that would ship every attribute in plaintext. +- **Committed source ships as written.** Masking applies to the documents, not to your repository: + a secret hardcoded in a `.tf` file reaches the platform even though the plan was masked. + `--no-source` is the opt-out. Terraform's `*_sensitive` markers are also not exhaustive — a + value that flows through `locals`, or comes from a provider that did not mark its schema, is not + caught by marker-driven masking. + +The number of masked values is printed before upload, and recorded in the bundle's metadata. + +## What it uploads + +One `tar.gz` archive per run, in the workflow's artifact directory, with a fixed layout: + +``` +plan.json the masked terraform plan +tfstate.json the masked state, if one was supplied +infracost.json the cost breakdown, if one was supplied +metadata.json what this bundle is: origin, repository, commit, masking, workflow identity +code/ the terraform source, if any was packed +``` + +The archive is retained after the run — it is the source that produced the findings, and other +systems read it to see the code a verdict came from. When a state document is supplied, the masked +copy is additionally published as the workflow's `tfstate.json` artifact so it appears in the +platform's State view; that copy is masked and cannot be used to run terraform. The full +`metadata.json` field reference is in +[docs/platform-check.md](https://github.com/StackGuardian/tirith/blob/main/docs/platform-check.md) +in the repository. + +## What it reports back + +- **Progress and the verdict headline go to stderr**, so stdout stays clean for machine-readable + output: the masking count, the upload, a link to the created run, each poll of the run's status, + and finally a one-line headline such as `Tirith — 3 failed, 1 warned`. +- **`--output-json`** writes the result document: the run `status`, the `verdict` + (`passed` | `warned` | `failed` | `no-policies` | `errored`), per-outcome `counts` (passed, + failed, warned, approval_required, skipped, unknown), the `headline`, `wfrun_id` and `wfrun_url` + linking to the run, the full `policy_results`, the `monthly_cost` when a cost breakdown was + evaluated, and where the uploaded archive lives (`archive_key`, `source_packed`, + `source_skipped_reason`). It is written once with `status: RUNNING` as soon as the run is + created — so a timeout still leaves the run discoverable — and again with the final result. +- **`--output-markdown`** writes a rendered report, suitable for posting as a pull-request or + merge-request comment by a later CI step. `--comment-marker` sets an opaque first line so your + script can find and update its own previous comment, and `--markdown-limit` truncates the body + (default 60000 characters). +- **The exit code** follows the shared [contract](exit-codes.md): `0` passed, `1` the check could + not be completed, `3` a policy failed (only with `--fail-on-error`), `130` interrupted. A run + that produced no verdict — errored, unreachable, unreadable results — always exits non-zero + regardless of `--fail-on-error`: it fails closed. A policy that asks for approval is reported as + a warning and does not block, because the evaluation has already finished by the time the intent + is known. + +## Credentials + +`--api-key` / `$SG_API_TOKEN` and `--org` / `$SG_ORG` are required. The key should be an +**organization** (`sgo_`) token — `sgu_` user tokens are non-functional for SSO-group-only users +and are warned about rather than rejected, so the symptom is a later 403. + +`--api-key -` reads the key from stdin, which keeps it out of the process table and out of shell +history: + +```sh +echo "$SG_TOKEN" | tirith platform check --api-key - --workflow-id infra +``` + +## Flag reference + +### Identity + +| Flag | Default | What it does | +|---|---|---| +| `--api-key` | `$SG_API_TOKEN` | API key, or `-` to read it from stdin | +| `--org` | `$SG_ORG` | Organization name | +| `--region` | `$SG_REGION` or `eu` | StackGuardian region, `eu` or `us`. Sets both the API and dashboard URLs at once | +| `--api-url` | `$SG_BASE_URL` | API base URL, with or without `/api/v1`. Overrides `--region`; needed only for a self-hosted install or a dedicated host | +| `--dashboard-url` | `$SG_DASHBOARD_URL` | Dashboard base URL, used to build run links. Inferred from `--api-url` when it names a known region | + +`--region` and an explicit URL cannot be combined — they set the same thing, and silently picking +one would hide the contradiction. + +### Workflow + +| Flag | Default | What it does | +|---|---|---| +| `--workflow-id` | *(required)* | Slug identifying the StackGuardian workflow. Created if absent. Letters, digits, `-` and `_` only; anything else is rejected with a suggested slug | +| `--workflow-group` | `default` | Workflow group. Created if absent — note that policies are scoped per group, so a typo silently enforces nothing | +| `--terraform-version` | | Stored on the workflow at creation | +| `--repo-url` | | Source repository URL, recorded on the workflow at creation so it links back to the code. Any credential embedded in the URL is stripped before it is recorded | +| `--repo-ref` | | Branch, tag or commit, recorded alongside `--repo-url` | +| `--repo-path` | inferred | Path of `--source-dir` within the repository, recorded in the bundle's `metadata.json`. Inferred from the enclosing git checkout if omitted | +| `--step-template-id` | platform default | Override the policy-evaluation step template | + +Runs on one workflow serialize while another is pending — a matrix that shares an id becomes a +queue, so give each leg its own. + +### Inputs + +| Flag | Default | What it does | +|---|---|---| +| `--input-path` | `plan.json` / `tfplan.json` in `--source-dir` | Document to evaluate | +| `--plan-file` | | Binary plan from `terraform plan -out=`. Rendered with `terraform show -json` in memory, so no unmasked plan JSON is ever written to disk. Cannot be combined with `--input-path` | +| `--terraform-bin` | auto-detected | terraform/tofu binary for `--plan-file`, preferring the real binary over a CI wrapper | +| `--input-kind` | `terraform_plan` | One of `terraform_plan`, `terraform_state`, `kubernetes`, `json`. Decides how the document is masked | +| `--state-path` | | Optional terraform state, masked before upload | +| `--infracost-path` | | Optional `infracost breakdown --format json` document | +| `--source-dir` | `.` | Terraform source to pack alongside the documents | +| `--no-source` | | Send only the documents. Discovery still looks in `--source-dir` (or `.`) for the plan | + +### Run + +| Flag | Default | What it does | +|---|---|---| +| `--sha` | | Commit SHA, used to namespace the uploaded archive | +| `--artifact-tag` | `default` | Namespaces the archive within a commit. Needed only when one workflow evaluates the same commit more than once — a plan phase and a state phase, or matrix legs sharing a workflow | +| `--trigger-details-json` | `{"type": "cli"}` | JSON object describing what triggered this run | +| `--trigger-details-file` | | File containing that JSON object | +| `--timeout` | `1800` | Seconds to wait for the run | + +### Output + +| Flag | Default | What it does | +|---|---|---| +| `--output-json` | | Write the result document here | +| `--output-markdown` | | Write a markdown report here | +| `--comment-marker` | | Opaque first line of the markdown, so a script can find its own comment | +| `--markdown-limit` | `60000` | Truncate the markdown to this length | +| `--fail-on-error` | off | Exit non-zero when a policy fails. An unreachable platform or a run that produced no verdict always exits non-zero regardless of this flag | diff --git a/documentation/docusaurus.config.js b/documentation/docusaurus.config.js index 980a086b..afa84dc2 100644 --- a/documentation/docusaurus.config.js +++ b/documentation/docusaurus.config.js @@ -5,10 +5,10 @@ const config = { title: 'Tirith', favicon: 'img/tirith.png', // Set the production url of your site here - url: 'https://your-docusaurus-site.example.com', - // Set the // pathname under which your site is served - // For GitHub pages deployment, it is often '//' - baseUrl: '/', + url: 'https://stackguardian.github.io', + // Set the // pathname under which your site is served. + // This is a GitHub Pages project site, so it is served under //. + baseUrl: '/tirith/', // GitHub pages deployment config. // If you aren't using GitHub pages, you don't need these. @@ -45,11 +45,12 @@ const config = { navbar: { title: 'Tirith', hideOnScroll: true, + // No href: the logo and title link to the site home, which is what a + // reader clicking a site's own logo expects. It used to open the policy + // builder in a new tab, which left no way back to the docs home. logo: { alt: 'Tirith Logo', src: 'img/tirith.png', - href: 'https://tirith-policy-builder.vercel.app/', - target:'_blank', }, items: [ { @@ -58,6 +59,11 @@ const config = { position: 'left', label: 'Docs', }, + { + href: 'https://tirith-policy-builder.vercel.app/', + label: 'Policy Builder', + position: 'right', + }, { href: 'https://github.com/StackGuardian/tirith', label: 'GitHub', diff --git a/documentation/sidebars.js b/documentation/sidebars.js index 752eb712..5638a35c 100644 --- a/documentation/sidebars.js +++ b/documentation/sidebars.js @@ -15,6 +15,17 @@ module.exports = { 'tirith-installation/manual-installation' ] }, + { + type: "category", + collapsed: true, + label: "Using Tirith", + items: [ + "tirith-usage/cli-reference", + "tirith-usage/exit-codes", + "tirith-usage/ci-integration", + "tirith-usage/platform-check", + ] + }, { type: "category", collapsed: true, @@ -22,11 +33,35 @@ module.exports = { items: [ "tirith-policies/tirith-create-first-policy", "tirith-policies/tirith-policy-structure", + "tirith-policies/tirith-policy-reference", "tirith-policies/tirith-policy-error-tolerance", "tirith-policies/tirith-policy-conditions", "tirith-policies/tirith-policy-variables", + "tirith-policies/tirith-policy-cookbook", // "tirith-policies/tirith-policy-examples" ] }, + { + type: "category", + collapsed: true, + label: "Providers", + items: [ + "tirith-providers/providers-overview", + "tirith-providers/terraform-plan-provider", + "tirith-providers/infracost-provider", + "tirith-providers/json-provider", + "tirith-providers/kubernetes-provider", + "tirith-providers/sg-workflow-provider", + ] + }, + { + type: "category", + collapsed: true, + label: "Reference", + items: [ + "tirith-reference/evaluators", + "tirith-reference/eval-expressions", + ] + }, ], -}; \ No newline at end of file +}; diff --git a/documentation/src/components/HomepageFeatures/index.js b/documentation/src/components/HomepageFeatures/index.js deleted file mode 100644 index 5488d344..00000000 --- a/documentation/src/components/HomepageFeatures/index.js +++ /dev/null @@ -1,60 +0,0 @@ -import clsx from 'clsx'; -import Heading from '@theme/Heading'; -import styles from './styles.module.css'; - -const FeatureList = [ - // { - // title: 'Easy to Use', - // Svg: require('@site/static/img/undraw_docusaurus_mountain.svg').default, - // description: ( - // <> - // Docusaurus was designed from the ground up to be easily installed and - // used to get your website up and running quickly. - // - // ), - // }, - // { - // title: 'Focus on What Matters', - // Svg: require('@site/static/img/undraw_docusaurus_tree.svg').default, - // description: ( - // <> - // Docusaurus lets you focus on your docs, and we'll do the chores. Go - // ahead and move your docs into the docs directory. - // - // ), - // }, - // { - // title: 'Powered by React', - // Svg: require('@site/static/img/undraw_docusaurus_react.svg').default, - // description: ( - // <> - // Extend or customize your website layout by reusing React. Docusaurus can - // be extended while reusing the same header and footer. - // - // ), - // }, -]; - -function Feature({title, description}) { - return ( -
-
- {title} -
-
- ); -} - -export default function HomepageFeatures() { - return ( -
-
-
- {FeatureList.map((props, idx) => ( - - ))} -
-
-
- ); -} diff --git a/documentation/src/components/HomepageFeatures/styles.module.css b/documentation/src/components/HomepageFeatures/styles.module.css deleted file mode 100644 index b248eb2e..00000000 --- a/documentation/src/components/HomepageFeatures/styles.module.css +++ /dev/null @@ -1,11 +0,0 @@ -.features { - display: flex; - align-items: center; - padding: 2rem 0; - width: 100%; -} - -.featureSvg { - height: 200px; - width: 200px; -} diff --git a/documentation/src/pages/index.js b/documentation/src/pages/index.js index 378ab87d..4cdf434b 100644 --- a/documentation/src/pages/index.js +++ b/documentation/src/pages/index.js @@ -1,32 +1,250 @@ -import clsx from 'clsx'; import Link from '@docusaurus/Link'; -import useDocusaurusContext from '@docusaurus/useDocusaurusContext'; import Layout from '@theme/Layout'; -import HomepageFeatures from '@site/src/components/HomepageFeatures'; - import Heading from '@theme/Heading'; +import CodeBlock from '@theme/CodeBlock'; + import styles from './index.module.css'; -function HomepageHeader() { - const {siteConfig} = useDocusaurusContext(); +/* + * --------------------------------------------------------------------------- + * COPY + * + * All prose for the landing page lives in this one object, deliberately kept + * apart from the markup below so it can be edited or lifted out without + * reading any JSX. + * + * It is derived from the repository README, which is the source of truth. If + * the two disagree, the README wins and this file is stale. + * --------------------------------------------------------------------------- + */ +const content = { + hero: { + title: 'Tirith — IaC Governance plugin', + tagline: + 'Plugin IaC Governance for any pipeline, running anywhere. Evaluate plans with Tirith, ' + + 'protect sensitive values, enforce centralised governance, and surface actionable results ' + + 'before infrastructure changes are applied.', + body: + 'Tirith reads the plan your pipeline already produces, checks it against your policies, and ' + + 'exits non-zero so a violating change never reaches apply. Apache-2.0, and no account needed.', + install: 'pip install git+https://github.com/StackGuardian/tirith.git', + actions: [ + {label: 'Get started', to: '/docs/getting-started-with-tirith/', primary: true}, + {label: 'GitHub', href: 'https://github.com/StackGuardian/tirith'}, + ], + }, + + problem: { + heading: 'The problem', + body: + 'A pipeline that runs init, plan and apply deploys whatever the plan says. Nothing sits ' + + 'between the plan and the change.', + points: [ + 'Every repository does it its own way, so there is no one place to see what was deployed, or what was refused.', + 'Rules that do exist live in whichever pipeline someone wrote them into, and get copied into the next repository by hand.', + 'When a check does fail, the log says a job failed. It does not say which rule, on which resource, or what value broke it.', + ], + }, + + add: { + heading: 'What you add', + body: 'Two lines, on GitHub Actions:', + code: + '- run: terraform show -json tfplan > plan.json\n' + + '- uses: StackGuardian/tirith-iac-governance-action@v2', + note: + 'With a plan.json in the working directory that is the whole integration — no with: block. ' + + 'Policies are JSON files committed under .tirith/policies.', + }, + + get: { + heading: 'What you get', + items: [ + { + title: 'Policies as data, not code', + body: + 'A rule is a JSON file describing what to look for, rather than a program you have to ' + + 'maintain. Terraform plans, terraform state, Kubernetes manifests, Infracost breakdowns ' + + 'and arbitrary JSON are all evaluated the same way.', + }, + { + title: 'Cost, before the change is applied', + body: + 'Point Tirith at an infracost breakdown and gate on the monthly or hourly total of the ' + + 'resources the plan would create.', + }, + { + title: 'Sensitive values masked on your own runner', + body: + 'Masking happens before anything leaves the machine, so a value marked sensitive stays ' + + 'out of the report and out of any upload.', + }, + { + title: 'An exit code your pipeline can act on', + body: + 'Exit 3 means a policy said no; exit 1 means Tirith could not tell you either way. A job ' + + 'that treats every non-zero code alike cannot tell a working gate from a broken one.', + }, + { + title: 'The plan and the code, kept together', + body: + 'In platform mode each run uploads the masked documents alongside the terraform source ' + + 'they describe, so a finding can still be read against the code that caused it later on.', + }, + { + title: 'One policy set, many pipelines', + body: + 'Because Tirith is a CLI rather than an integration built into one CI system, the same ' + + 'policies gate a GitHub Actions job, a GitLab job and a laptop. In platform mode, Tirith ' + + 'rules and Checkov findings come back in a single verdict.', + }, + ], + }, + + worksWith: { + heading: 'Works with', + items: [ + { + title: 'GitHub Actions', + body: + 'A native action that finds the plan, posts a sticky pull-request comment, creates a ' + + 'check run and sets the exit code.', + link: { + label: 'tirith-iac-governance-action', + href: 'https://github.com/StackGuardian/tirith-iac-governance-action', + }, + }, + { + title: 'GitLab CI, and any container-based CI', + body: + 'Install the CLI in the job and call it directly, which is all the action does ' + + 'underneath. There is no GitLab-native equivalent of the action.', + }, + { + title: 'Your machine', + body: 'The same command, the same verdict, no account and no network.', + }, + ], + }, + + platform: { + heading: 'Keeping policy in one place', + body: + 'Everything above works with policy files committed to your repository. If you would rather ' + + 'not copy those files into every repository that needs gating, tirith platform check ' + + 'evaluates against the policies a StackGuardian organization enforces instead — same ' + + 'document, same verdict, same exit codes, plus a central run history. That mode is optional, ' + + 'and is the only part that talks to a network.', + link: {label: 'Read about platform mode', to: '/docs/tirith-usage/platform-check/'}, + }, +}; + +/* + * --------------------------------------------------------------------------- + * MARKUP + * --------------------------------------------------------------------------- + */ + +// Uses Docusaurus's own button classes rather than hand-rolled ones: they carry +// a readable foreground in both light and dark mode. A custom rule here had set +// the label to var(--ifm-background-color), which is #0000 in light mode -- so +// the text was transparent on a purple fill. +function Action({label, to, href, primary}) { + const className = `button button--lg ${primary ? 'button--primary' : 'button--secondary'}`; + return to ? ( + + {label} + + ) : ( + + {label} + + ); +} + +function Hero() { + const {title, tagline, body, install, actions} = content.hero; return ( -
- - {siteConfig.title} - +
+ + {title} + +

{tagline}

+

{body}

+ {install} +
+ {actions.map((action) => ( + + ))}
+
+ ); +} + +function Section({heading, children}) { + return ( +
+ + {heading} + + {children} +
); } export default function Home() { - const {siteConfig} = useDocusaurusContext(); return ( - - -
- + +
+ + +
+

{content.problem.body}

+
    + {content.problem.points.map((point) => ( +
  • {point}
    • + ))} +
+
+ +
+

{content.add.body}

+ {content.add.code} +

{content.add.note}

+
+ +
+
    + {content.get.items.map((item) => ( +
  • + {item.title}. {item.body} +
    • + ))} +
+
+ +
+
    + {content.worksWith.items.map((item) => ( +
  • + {item.title} — {item.body} + {item.link ? ( + <> + {' '} + {item.link.label}. + + ) : null} +
    • + ))} +
+
+ +
+

{content.platform.body}

+

+ {content.platform.link.label} +

+
); diff --git a/documentation/src/pages/index.module.css b/documentation/src/pages/index.module.css index 9f71a5da..6154747a 100644 --- a/documentation/src/pages/index.module.css +++ b/documentation/src/pages/index.module.css @@ -1,23 +1,64 @@ /** - * CSS files with the .module.css suffix will be treated as CSS modules - * and scoped locally. + * Landing page styles. Kept deliberately small: this page is a placeholder, and + * it should not grow a design system that the real rebuild would have to undo. + * + * Colours come from Docusaurus theme variables so light and dark mode both work + * without a second palette being defined here. */ -.heroBanner { - padding: 4rem 0; - text-align: center; - position: relative; - overflow: hidden; +.page { + max-width: 46rem; + margin: 0 auto; + padding: 3rem 1.25rem 5rem; } -@media screen and (max-width: 996px) { - .heroBanner { - padding: 2rem; - } +.hero { + margin-bottom: 1rem; +} + +.heroTitle { + font-size: 2.25rem; + letter-spacing: -0.02em; + margin-bottom: 0.75rem; +} + +.tagline { + font-size: 1.15rem; + margin-bottom: 1rem; +} + +.muted { + color: var(--ifm-color-emphasis-700); +} + +.section { + margin-top: 3rem; + padding-top: 1.5rem; + border-top: 1px solid var(--ifm-color-emphasis-300); } -.buttons { +.sectionHeading { + font-size: 1.25rem; + margin-bottom: 0.75rem; +} + +.list li { + margin-bottom: 0.75rem; +} + +.actions { display: flex; - align-items: center; - justify-content: center; + flex-wrap: wrap; + gap: 0.75rem; + margin-top: 1.5rem; +} + +@media screen and (max-width: 996px) { + .page { + padding: 2rem 1rem 3rem; + } + + .heroTitle { + font-size: 1.75rem; + } } diff --git a/setup.py b/setup.py index 7d07cb9a..667e0b5a 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ def read(*names, **kwargs): setup( name="py-tirith", - version="1.0.5", + version="1.2.0", license="Apache", description="Tirith simplifies defining Policy as Code.", long_description_content_type="text/markdown", diff --git a/src/tirith/__init__.py b/src/tirith/__init__.py index 151dee52..4c2aac77 100644 --- a/src/tirith/__init__.py +++ b/src/tirith/__init__.py @@ -2,6 +2,6 @@ tirith: Execute policies defined using Tirith (StackGuardian Policy Framework) """ -__version__ = "1.0.5" +__version__ = "1.2.0" __author__ = "StackGuardian" __license__ = "Apache" diff --git a/src/tirith/cli.py b/src/tirith/cli.py index 6642e312..f08c85e8 100755 --- a/src/tirith/cli.py +++ b/src/tirith/cli.py @@ -15,7 +15,6 @@ from .core import start_policy_evaluation - logger = logging.getLogger(__name__) @@ -27,6 +26,23 @@ def eprint(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) +# Subcommands are dispatched before the flat parser sees anything. argparse cannot express an +# optional subcommand alongside options like `-policy-path` (a single dash and a long name), and the +# local-evaluation surface is a contract: tests/core/test_output_compatibility.py asserts its --json +# output is byte-identical to a golden file. An explicit pre-dispatch leaves that untouched. +# +# Named `platform` because that is what it evaluates against: the policies your StackGuardian +# organization enforces, run on the platform, rather than policy files in your repository. +# +# It was briefly `remote` on this branch, on the argument that "platform check" can read as *a check of +# the platform*. Reverted -- the vagueness is minor next to having one name, and the concern that +# prompted the rename was really that the open-source surface could not gate at all, which +# `--fail-on-error` fixed. No alias in either direction: nothing is released, so there is no caller to +# keep working. +SUBCOMMAND = "platform" +SUBCOMMANDS = {SUBCOMMAND} + + def main(args=None) -> ExitStatus: """ The main function. @@ -36,6 +52,13 @@ def main(args=None) -> ExitStatus: Return exit status code. """ + argv = list(sys.argv[1:] if args is None else args) + + if argv and argv[0] in SUBCOMMANDS: + from tirith.platform import cli as platform_cli + + return platform_cli.main(argv) + try: class _WidthFormatter(argparse.RawTextHelpFormatter): @@ -47,6 +70,11 @@ def __init__(self, prog="PROG") -> None: formatter_class=_WidthFormatter, epilog=textwrap.dedent( """\ + Subcommands: + + tirith platform check --help Evaluate against the policies your StackGuardian + organization enforces, rather than local files. + About Tirith: * Abstract away the implementation complexity of policy engine underneath. @@ -54,7 +82,7 @@ def __init__(self, prog="PROG") -> None: * Provide a standard framework for scanning various configurations with granularity. * Provide modularity to enable easy extensibility * Github - https://github.com/StackGuardian/tirith - * Docs - https://docs.stackguardian.io/docs/tirith/overview + * Docs - https://github.com/StackGuardian/tirith#readme """ ), ) @@ -102,11 +130,17 @@ def __init__(self, prog="PROG") -> None: action="store_true", help="Show detailed logs of from the run", ) + parser.add_argument( + "--fail-on-error", + dest="failOnError", + action="store_true", + help="Exit 3 when a policy fails, instead of 0. Off by default for compatibility.", + ) parser.add_argument("--version", action="version", version=__version__) - args = parser.parse_args() + args = parser.parse_args(argv) - if len(sys.argv) == 1: + if not argv: parser.print_help() sys.exit(0) @@ -135,6 +169,49 @@ def __init__(self, prog="PROG") -> None: print(formatted_result) else: pretty_print_result_dict(result) + + # Without --fail-on-error this returns 0 whether the policy passed or failed, which is + # what it has always done: the verdict is in the output, and changing that silently would + # turn every existing green CI job red on upgrade. + # + # But a gate that cannot fail is not a gate, and this was the only way to run tirith + # without an account -- so the honest answer was an opt-in flag rather than pointing + # people at the hosted path when they need an exit code that means something. + # + # 3, not 1, and the distinction is the point: 3 says the infrastructure violates a policy, + # 1 says tirith could not tell you. The same split `platform check` uses, because a caller + # scripting both should not have to learn two vocabularies. + # + # `final_result` is tri-state, and that is what decides: + # + # True every check that ran passed -> 0 + # False a check ran and said no -> 3 + # None nothing ran; every check was skipped -> 1 + # absent the policy could not be loaded at all -> 1 + # + # None is not a pass. A policy whose every check was skipped -- `error_tolerance` swallowing + # a provider that found nothing -- checked precisely nothing, and reporting that as green is + # the failure this whole flag exists to prevent. `absent` is the missing-variables path, + # which returns `errors` and no result at all. + # + # `errors` is deliberately NOT consulted. It reads like a tool-failure signal and is not: it + # is populated only by the eval-expression pass, and the one thing that puts a message there + # beside a real verdict is the informational "these ids are not defined and have been + # removed" note. Gating on it inverted both halves of this contract -- a genuine violation + # whose expression mentioned a typo'd id exited 1, while a policy naming an unknown provider + # exited 3. + # + # Known limit, worth stating rather than pretending otherwise: a *misconfigured* policy -- an + # unsupported `condition.type`, an unknown `required_provider` -- surfaces from the engine as + # an ordinary failed evaluator with no error attached, so it is indistinguishable from a + # violation here and exits 3. Fixing that means the engine reporting it distinctly, not this + # branch guessing from free text. + if args.failOnError: + final_result = result.get("final_result") + if "final_result" not in result or final_result is None: + return ExitStatus.ERROR + if final_result is not True: + return ExitStatus.ERROR_POLICY_FAILED return ExitStatus.SUCCESS except Exception as e: # TODO:write an exception class for all provider exceptions. diff --git a/src/tirith/core/core.py b/src/tirith/core/core.py index 27c60646..0dfedaa5 100644 --- a/src/tirith/core/core.py +++ b/src/tirith/core/core.py @@ -12,7 +12,6 @@ from .evaluators import EVALUATORS_DICT from .policy_parameterization import get_policy_with_vars_replaced - logger = logging.getLogger(__name__) @@ -50,6 +49,10 @@ def generate_evaluator_result(evaluator_obj, input_data, provider_module): evaluator_class = EVALUATORS_DICT.get(evaluator_name) if evaluator_class is None: logger.error(f"{evaluator_name} is not a supported evaluator") + # Always populate "result" before returning. Consumers (the pretty printer, the + # workflow-step templates, the platform) index into it unconditionally, and an + # early return without it used to raise KeyError far away from the real cause. + result["result"] = [{"passed": False, "message": f"`{evaluator_name}` is not a supported evaluator"}] return result evaluator_instance = evaluator_class() @@ -66,6 +69,17 @@ def generate_evaluator_result(evaluator_obj, input_data, provider_module): has_valid_evaluation = False for evaluator_input in evaluator_inputs: + # A provider reported an error without attaching a ProviderError severity. That means a + # malformed provider call -- an unsupported operation_type, a missing required argument -- + # not a policy violation. Surface the message and fail hard: error_tolerance exists to + # tolerate missing data, never to mask a broken policy. Without this branch the error text + # is discarded and `None` is evaluated against the condition, so a typo'd operation_type + # reads as a genuine violation. + if evaluator_input.get("err") and not isinstance(evaluator_input["value"], ProviderError): + evaluation_results.append({"passed": False, "message": evaluator_input["err"]}) + has_evaluation_passed = False + continue + if isinstance(evaluator_input["value"], ProviderError) and evaluator_input.get("err", None): severity_value = evaluator_input["value"].severity_value err_result = dict(message=evaluator_input["err"]) @@ -139,6 +153,18 @@ def visit_UnaryOp(self, node: ast.UnaryOp) -> Any: tree = ast.parse(eval_str, mode="eval") + # `&` and `|` parse as BinOp, which nothing below handles: the tree stays uncompilable, the retry + # loop exhausts, and the caller reports "Could not evaluate the eval expression. Please report this + # error" -- telling a user to file a bug against their own typo. The README documented `&` in two + # examples, so this was reachable by copying the docs. Name the operator instead. + for node in ast.walk(tree): + if isinstance(node, ast.BinOp): + operators = {ast.BitAnd: ("&", "&&"), ast.BitOr: ("|", "||")} + wrong, right = operators.get(type(node.op), (None, None)) + if wrong: + raise ValueError(f"Unsupported operator '{wrong}' in eval_expression. Use '{right}' instead.") + raise ValueError("Unsupported operator in eval_expression. Only '&&', '||' and '!' are supported.") + compiled_code = None tries_count = 0 is_tree_compilable = False @@ -302,8 +328,16 @@ def start_policy_evaluation_from_dict(policy_dict: Dict, input_dict: Dict, var_d eval_results.append(eval_result) final_evaluation_result, errors = final_evaluator(final_evaluation_policy_string, eval_results_obj) + # Pass policy-declared metadata through to the result, but only the keys that are actually + # present. Absent keys are omitted rather than emitted as null, so the output of a policy + # that declares none of them is byte-identical to what it was before this was added. + final_output_meta = {"version": policy_meta.get("version"), "required_provider": provider_module} + for meta_key in ("id", "name", "description", "severity", "enforcement", "tags", "remediation"): + if meta_key in policy_meta: + final_output_meta[meta_key] = policy_meta[meta_key] + final_output = { - "meta": {"version": policy_meta.get("version"), "required_provider": provider_module}, + "meta": final_output_meta, "final_result": final_evaluation_result, "evaluators": eval_results, "errors": errors, diff --git a/src/tirith/core/policy_parameterization.py b/src/tirith/core/policy_parameterization.py index ce81dafe..c34092af 100644 --- a/src/tirith/core/policy_parameterization.py +++ b/src/tirith/core/policy_parameterization.py @@ -1,3 +1,4 @@ +import copy import re import pydash @@ -52,11 +53,17 @@ def get_policy_with_vars_replaced(policy_dict: dict, var_dict: dict) -> Tuple[di """ Replace the variables in the policy_dict with the values from the var_dict + The caller's `policy_dict` is never mutated: substitution happens on a deep copy. This + matters when the same parsed policy is evaluated more than once (for example a policy set + run against several inputs, or a retry), where substituted values would otherwise leak + from one evaluation into the next. + :param policy_dict: The policy dictionary :param var_dict: The dictionary containing the variables - :return: The policy dictionary with the variables replaced + :return: A copy of the policy dictionary with the variables replaced and the list of variables that are not found """ + policy_dict = copy.deepcopy(policy_dict) not_found_vars = [] # Replace vars in the meta key _replace_vars_in_dict(policy_dict["meta"], var_dict, not_found_vars) diff --git a/src/tirith/platform/__init__.py b/src/tirith/platform/__init__.py new file mode 100644 index 00000000..ae9467ba --- /dev/null +++ b/src/tirith/platform/__init__.py @@ -0,0 +1,6 @@ +""" +StackGuardian platform integration. + +Everything here is stdlib-only on purpose: tirith has three runtime dependencies and none of them +are an HTTP library, so a CI runner needs nothing installed beyond tirith itself. +""" diff --git a/src/tirith/platform/archive.py b/src/tirith/platform/archive.py new file mode 100644 index 00000000..e832328a --- /dev/null +++ b/src/tirith/platform/archive.py @@ -0,0 +1,372 @@ +""" +Build the gzipped tar that carries a run's inputs to StackGuardian. + +The step unpacks this and reads the documents out of it, so the layout is a contract: + + plan.json terraform plan JSON -- the primary policy input + tfstate.json terraform state JSON + infracost.json cost breakdown + metadata.json what this bundle is: repository, commit, where the code belongs + code/ the terraform source, if any was packed + +**The documents stay at the root.** The step joins those three names onto the extraction directory and +treats absence as normal -- so moving one under a prefix would not raise, it would make every policy +report "unevaluated" and the run would look like it passed with warnings. + +**`code/` is a prefix, not a directory member.** Nothing writes an explicit directory entry, so the +prefix exists in the tar only while at least one file carries it. `metadata.json` says so rather than +leaving a consumer to infer it from an absence. + +Two things here are easy to get wrong and expensive to get wrong. + +**The masked documents go in, never the originals.** `pack()` takes already-redacted objects and +serializes them itself; it never copies plan.json off disk. A caller that packed the source +directory *first* and masked afterwards would ship the plaintext file alongside the masked one. The +tests assert on the bytes inside the resulting tarball for this reason -- asserting on the dict +that was passed in would pass while the archive leaked. + +**`.terraform/` must be excluded.** A provider cache is routinely hundreds of megabytes; including +it would make every run upload the AWS provider. `*.tfstate*` is excluded for the same reason as +the first point: an unmasked state file sitting in the working directory would otherwise travel +next to the masked copy. +""" + +import fnmatch +import io +import os +import posixpath +import tarfile + +# Fixed names the step looks for at the archive root. +PLAN_DOCUMENT = "plan.json" +STATE_DOCUMENT = "tfstate.json" +INFRACOST_DOCUMENT = "infracost.json" + +# What this bundle is, for whatever reads it later. Written at the root beside the documents. +METADATA_DOCUMENT = "metadata.json" + +# The source tree lives under here, so the root belongs to us alone. +CODE_PREFIX = "code" + +# These names are ALWAYS written by pack(), never copied from the source tree -- whether or not a +# masked document was supplied for them. +# +# This is a LEAK guard, not a collision guard, and the distinction matters now that the source sits +# under `code/` where it cannot collide with anything. A file called tfstate.json in the working +# directory is raw, unmasked state by definition -- `terraform state pull > state.json` is the +# documented way to make one -- so packing it as `code/tfstate.json` would ship every attribute in +# plaintext next to the masked copy. Nothing about the prefix makes that safe; see the note in pack(). +# +# metadata.json is here for a plainer reason: it is a thoroughly ordinary filename for a repository to +# contain, tar tolerates duplicate members, and extraction order would decide which one won. +RESERVED_DOCUMENTS = frozenset((PLAN_DOCUMENT, STATE_DOCUMENT, INFRACOST_DOCUMENT, METADATA_DOCUMENT)) + +# Always excluded, regardless of .gitignore. +# +# .terraform/ provider binaries and modules; hundreds of MB, and the runner does its own init +# .git/ full history, so anything ever committed would ship +# *.tfstate* raw state -- unmasked by definition, including .backup files +# tfplan / *.tfplan the BINARY plan. It embeds the prior state, so it carries every attribute of +# every existing resource in plaintext -- strictly worse than a raw state file, +# and it matches none of the *.tfstate patterns. `--plan-file` reads it, converts +# it and masks the result in memory, which the source walk then undid by packing +# the original. +# .terraform.lock.hcl is deliberately NOT excluded: it pins provider versions and is small. +DEFAULT_EXCLUDES = ( + ".git", + ".terraform", + "*.tfstate", + "*.tfstate.*", + "tfplan", + "*.tfplan", + "*.tfplan.*", + "__pycache__", + "*.pyc", + ".venv", + "node_modules", +) + +# Refuse to build anything larger than this. A runaway archive is nearly always an exclusion that +# did not fire, and failing loudly beats a five-minute upload that times out the run. +# +# Overridable, because the source tree is packed by default and the only other lever is dropping it +# entirely: a large monorepo that genuinely needs to ship its code has nowhere else to go. Raising it +# trades a clear error for a slow upload and more memory on the runner -- the whole archive is built +# in memory before this is checked -- so it is deliberately not a documented headline. +MAX_ARCHIVE_BYTES = 100 * 1024 * 1024 + +_override = os.environ.get("TIRITH_MAX_ARCHIVE_BYTES", "").strip() +if _override: + try: + MAX_ARCHIVE_BYTES = int(_override) + except ValueError: + # Not worth failing a run over; the default is a safe answer. + pass + + +class ArchiveError(Exception): + """The archive could not be built.""" + + +def _human_bytes(count): + """ + A size a person can read. + + Integer MB division reported anything under a megabyte as "0 MB", which is what the size limit + message used to say -- and that message is now surfaced on a pull request, where "0 MB over the + 0 MB limit" tells the reader nothing. + """ + for unit, size in (("MB", 1024 * 1024), ("KB", 1024)): + if count >= size: + return f"{count / size:.1f} {unit}" + return f"{count} bytes" + + +def _load_gitignore_patterns(source_dir): + """ + Read .gitignore into fnmatch patterns. + + Deliberately simple: leading `/` and trailing `/` are stripped, negations (`!`) are ignored. + A full gitignore implementation is not worth it here -- DEFAULT_EXCLUDES covers the cases that + actually matter, and .gitignore is a convenience on top. + """ + path = os.path.join(source_dir, ".gitignore") + patterns = [] + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or line.startswith("!"): + continue + patterns.append(line.strip("/")) + except OSError: + return [] + return patterns + + +def _is_excluded(relative_path, name, patterns): + """Match a path against the exclusion patterns, by both basename and full relative path.""" + for pattern in patterns: + if fnmatch.fnmatch(name, pattern) or fnmatch.fnmatch(relative_path, pattern): + return True + # A directory pattern excludes everything beneath it. + if relative_path.startswith(pattern + os.sep): + return True + return False + + +def pack( + source_dir, + plan=None, + state=None, + infracost=None, + extra_excludes=(), + respect_gitignore=True, + document_sources=(), + metadata=None, +): + """ + Build the archive in memory and return its bytes. + + `plan`, `state` and `infracost` are already-redacted objects. They are serialized here and + written at the archive root, overriding any same-named file in `source_dir` -- so a stale + plan.json lying around cannot displace the masked one. + + `metadata` is the caller's half of `metadata.json`: what it intended. This function fills in the + half only it can observe -- whether a tree was actually walked, under what prefix, and how many + files went in or were skipped -- and writes the member last. The split is deliberate: a bundle + that claims code but packed nothing is detectable only because the count is produced here rather + than asserted by the caller. + + `document_sources` are the paths those objects were *read from*. They are excluded from the + source walk, because the file on disk is the unmasked original: masking `tfplan.json` and then + packing the source tree shipped the plaintext copy one filename away from the redacted one. + Reserving only the three names this function writes was not enough -- the input is routinely + called something else (`tfplan.json`, `state.json`, or the binary `tfplan`, which carries the + prior state inside it). + + Returns (archive_bytes, manifest) where manifest lists what went in, for logging. + """ + if source_dir and not os.path.isdir(source_dir): + raise ArchiveError(f"Source directory does not exist: {source_dir}") + + patterns = list(DEFAULT_EXCLUDES) + list(extra_excludes) + if respect_gitignore and source_dir: + patterns += _load_gitignore_patterns(source_dir) + + documents = {} + if plan is not None: + documents[PLAN_DOCUMENT] = plan + if state is not None: + documents[STATE_DOCUMENT] = state + if infracost is not None: + documents[INFRACOST_DOCUMENT] = infracost + + buffer = io.BytesIO() + manifest = {"documents": sorted(documents), "files": 0, "skipped": 0} + + with tarfile.open(fileobj=buffer, mode="w:gz") as tar: + if source_dir: + # RESERVED_DOCUMENTS, not just the ones being written. A file named tfstate.json in the + # working directory is unmasked by definition -- `terraform state pull > state.json` is + # the documented way to produce one -- so packing it would ship every attribute in + # plaintext beside the masked copy. If the caller wants it evaluated they pass + # --state-path, which masks it first. + # + # Plus whatever the documents were actually read from, which is usually named something + # else entirely. + reserved = set(RESERVED_DOCUMENTS) | _relative_sources(source_dir, document_sources) + manifest["files"], manifest["skipped"] = _add_tree(tar, source_dir, patterns, reserved) + for name, document in documents.items(): + _add_document(tar, name, document) + if metadata is not None: + _add_document(tar, METADATA_DOCUMENT, _observed_metadata(metadata, source_dir, manifest)) + + archive = buffer.getvalue() + if len(archive) > MAX_ARCHIVE_BYTES: + raise ArchiveError( + f"Archive is {_human_bytes(len(archive))}, over the {_human_bytes(MAX_ARCHIVE_BYTES)} " + "limit. This usually means a large directory was not excluded -- check for provider " + "caches or build output, and pass extra excludes if needed." + ) + + manifest["bytes"] = len(archive) + return archive, manifest + + +def _observed_metadata(metadata, source_dir, manifest): + """ + Overlay what this module observed onto the caller's metadata, without mutating it. + + `code.present` is `files > 0`, not "a source directory was requested". Nothing writes an explicit + directory member, so a tree where every file was excluded leaves no `code/` in the tar at all -- + and a consumer comparing the two must not find them disagreeing. `present` therefore means + literally "there are members under the prefix". + + The caller's `code.absent_reason` survives when it has one (it knows *why* it asked for no source); + a tree that was requested and vanished into the exclude list gets one from here, because the caller + cannot know that happened. + """ + code = dict(metadata.get("code") or {}) + files = manifest.get("files", 0) + present = bool(source_dir) and files > 0 + + code["present"] = present + code["prefix"] = f"{CODE_PREFIX}/" if present else None + code["files"] = files + code["skipped"] = manifest.get("skipped", 0) + if not present: + code["repo_path"] = None + code["repo_path_from"] = None + if not code.get("absent_reason"): + code["absent_reason"] = "empty_after_excludes" if source_dir else "not_requested" + + merged = dict(metadata) + merged["code"] = code + merged["documents"] = { + "plan": PLAN_DOCUMENT if PLAN_DOCUMENT in manifest.get("documents", ()) else None, + "state": STATE_DOCUMENT if STATE_DOCUMENT in manifest.get("documents", ()) else None, + "infracost": INFRACOST_DOCUMENT if INFRACOST_DOCUMENT in manifest.get("documents", ()) else None, + } + return merged + + +def _add_tree(tar, source_dir, patterns, reserved_names, prefix=CODE_PREFIX): + """ + Walk `source_dir`, adding everything not excluded under `prefix`. Returns (added, skipped). + + Member names are built with `posixpath`, not `os.path`: tar names are `/`-separated on every + platform, and joining with the OS separator would emit backslashes on Windows -- extracting to + literal one-segment filenames with backslashes in them. + """ + added = 0 + skipped = 0 + + for root, dirs, files in os.walk(source_dir): + relative_root = os.path.relpath(root, source_dir) + relative_root = "" if relative_root == "." else relative_root + + # Prune in place so os.walk does not descend into excluded directories at all -- the point + # of excluding .terraform is not to read it. + kept_dirs = [] + for d in dirs: + relative = os.path.join(relative_root, d) if relative_root else d + if _is_excluded(relative, d, patterns): + skipped += 1 + elif os.path.islink(os.path.join(root, d)): + # os.walk does not follow symlinked directories, so this one contributes nothing -- + # count it rather than letting a whole subtree disappear without appearing anywhere in + # the manifest. Same reasoning as the file-level islink guard below. + skipped += 1 + else: + kept_dirs.append(d) + dirs[:] = kept_dirs + + for name in files: + relative = os.path.join(relative_root, name) if relative_root else name + if _is_excluded(relative, name, patterns): + skipped += 1 + continue + # Reserved names are skipped on the path they have in the SOURCE tree, before the prefix + # is applied. `code/tfstate.json` could not displace the masked root copy, but it would + # still be unmasked state inside the bundle -- which is the actual reason for this skip. + if relative in reserved_names: + skipped += 1 + continue + full = os.path.join(root, name) + if os.path.islink(full): + # A symlink out of the tree would either break on extraction or smuggle a file in. + skipped += 1 + continue + if not os.path.isfile(full): + # Sockets, fifos and device nodes. `tar.add` does not raise for a type it cannot + # classify -- it debug-logs "Unsupported type" and returns -- so counting the attempt + # made `added` disagree with what the tar actually holds, and `code.present` could + # then be true with nothing under the prefix at all. + skipped += 1 + continue + try: + tar.add(full, arcname=posixpath.join(prefix, relative.replace(os.sep, "/"))) + added += 1 + except OSError: + skipped += 1 + + return added, skipped + + +def _relative_sources(source_dir, document_sources): + """ + The document source paths, expressed the way _add_tree names members, for exclusion. + + Anything outside `source_dir` is dropped rather than kept as an unanchored basename: it cannot + collide with a member name, and excluding a bare basename would silently drop an unrelated + same-named file from the archive. + """ + relative = set() + try: + root = os.path.realpath(source_dir) + except OSError: + return relative + + for path in document_sources or (): + if not path: + continue + try: + full = os.path.realpath(path) + rel = os.path.relpath(full, root) + except (OSError, ValueError): + continue + if rel != os.pardir and not rel.startswith(os.pardir + os.sep) and not os.path.isabs(rel): + relative.add(rel) + return relative + + +def _add_document(tar, name, document): + """Serialize one document straight into the tar, never via a file on disk.""" + import json + + payload = document if isinstance(document, bytes) else json.dumps(document).encode("utf-8") + info = tarfile.TarInfo(name=name) + info.size = len(payload) + info.mode = 0o644 + tar.addfile(info, io.BytesIO(payload)) diff --git a/src/tirith/platform/check.py b/src/tirith/platform/check.py new file mode 100644 index 00000000..643bf627 --- /dev/null +++ b/src/tirith/platform/check.py @@ -0,0 +1,718 @@ +""" +Orchestration for `tirith platform check`. + + read -> mask -> pack -> ensure workflow -> upload archive -> create run -> poll -> fetch -> report + +The masking is the part that matters most and it happens *here*, on the caller's machine, before +anything leaves it. Masking server-side would be theatre: once the bytes arrive the exposure has +already happened. +""" + +import datetime +import json +import os +import posixpath +import sys +import urllib.parse + +from .. import __version__ +from . import archive, redact, report +from .client import ARCHIVE_DOCUMENT, ARCHIVE_NAME_TEMPLATE, SGClient, SGError + +# The version of the metadata.json contract. One integer, bumped only when a change breaks a reader; +# added fields do not bump it. A consumer seeing a higher number should read what it recognises and +# refuse to act destructively -- in particular, it must not write files back using `code.repo_path` +# from a schema it does not understand. +METADATA_SCHEMA_VERSION = 1 + +# Hosts we can name with confidence. Anything else is reported as `unknown` with the raw host +# alongside, because a self-hosted GitLab at git.example.internal is unrecognisable by design and +# guessing "github" for it would be worse than admitting ignorance. +_KNOWN_VCS_HOSTS = { + "github.com": "github", + "gitlab.com": "gitlab", + "bitbucket.org": "bitbucket", + "dev.azure.com": "azure_devops", + "ssh.dev.azure.com": "azure_devops", +} + +DEFAULT_WORKFLOW_GROUP = "default" +DEFAULT_TERRAFORM_VERSION = "1.5.7" + +# What the CLI understands as an input document. `terraform_state` exists as a distinct kind from +# `json` purely so this side knows to mask it -- tirith itself has no state provider, and the step +# routes it to the json provider. +INPUT_KINDS = ("terraform_plan", "terraform_state", "kubernetes", "json") + +# The bundle's name lives in client.ARCHIVE_NAME_TEMPLATE, and the reasoning is worth keeping here +# because it inverted twice while this was built. +# +# It began as `__sg.{sha}-{tag}.tar.gz`. The `__sg.` prefix deliberately kept it OUT of the artifact +# sync, because that prefix is pulled into every run's working directory and pushed back with no +# --delete. Once the sync became the *delivery* mechanism -- the step reads the bundle out of +# $LOCAL_ARTIFACTS_DIR -- being excluded from it was exactly wrong, so the name must match none of the +# sync's exclude patterns (`sg.*`, `*__sg.*`, `*pci_*`, the compliance globs) and must not be +# `tfstate.json`. +# +# The sha stays, though, and it is load-bearing. A name shared by every run of the workflow is a name +# two concurrent runs can overwrite -- and the action derives one workflow id per repository, so two +# open pull requests is the ordinary case, not a corner. One run would then evaluate the other's code +# and report the verdict as its own, silently, on a merge gate. Per commit, that cannot happen. +# +# It is affordable because the name is per *run*, not per workflow: core merges the run's +# TerraformConfig over the workflow's, so each run names its own bundle in its own +# `prePlanWfStepsConfig`. The workflow's stored copy is only a fallback. +# +# The cost is growth -- bundles accumulate in a prefix with no lifecycle rule, no --delete on either +# sync, and no artifact DELETE in api, so every later run downloads all of them. Taken deliberately: +# correctness over transfer cost. `client.delete_artifact` is kept for a retention sweep to use. + +# Deliberately NOT `__sg.`-prefixed, unlike the archive. This one is meant to be seen: it is the name +# the platform already treats as a workflow's state document, so it lands in the State and artifacts +# views rather than being hidden from them. The name is shared with the copy inside the archive +# (`archive.STATE_DOCUMENT`). +STATE_DOCUMENT_NAME = "tfstate.json" +STATE_CONTENT_TYPE = "application/json" + + +class CheckError(Exception): + """The check could not be completed. Always fails closed.""" + + +def log(message): + """Progress goes to stderr so stdout stays clean for machine-readable output.""" + print(message, file=sys.stderr, flush=True) + + +def read_json(path, label): + if not os.path.exists(path): + raise CheckError(f"{label} not found: {path}") + try: + with open(path, "r") as f: + return json.load(f) + except json.JSONDecodeError as e: + raise CheckError(f"{label} is not valid JSON ({path}): {e}") + except OSError as e: + raise CheckError(f"Could not read {label} ({path}): {e}") + + +def prepare_documents(input_path, input_kind, state_path, infracost_path, input_document=None): + """ + Read and mask everything that will go into the archive. + + Returns (plan, state, infracost, redaction_count). The returned objects are the *masked* ones; + nothing downstream should ever touch the originals again. + + `input_document` is an already-parsed document, used by --plan-file so `terraform show -json` + output goes straight from the pipe into the masker without an unmasked plan ever being written + to disk. + """ + plan = None + state = None + redactions = 0 + + if input_document is not None or input_path: + document = input_document if input_document is not None else read_json(input_path, "input document") + if input_kind == "terraform_plan": + plan = redact.redact_plan(document) + redactions += redact.count_redactions(plan) + elif input_kind == "terraform_state": + state = redact.redact_state(document) + redactions += redact.count_redactions(state) + else: + # kubernetes / json: no marker structure to drive masking, so it goes as-is. Warn if it + # looks like state, because that is the mistake that would ship every attribute in + # plaintext. + if isinstance(document, dict) and {"version", "lineage", "resources"} <= set(document): + log( + "WARNING: this document looks like terraform state but --input-kind is " + f"'{input_kind}', so it will NOT be masked. Use --input-kind terraform_state." + ) + plan = document + + if state_path: + state_document = read_json(state_path, "state document") + masked_state = redact.redact_state(state_document) + redactions += redact.count_redactions(masked_state) + if state is None: + state = masked_state + else: + log("Both --input-path and --state-path are state documents; using --input-path") + + infracost = read_json(infracost_path, "cost breakdown") if infracost_path else None + + return plan, state, infracost, redactions + + +# The step template that evaluates the policies, and the name its run stage takes. +POLICY_STEP_TEMPLATE = "/stackguardian/tirith-iac-governance:1" +# Names the run stage, so it surfaces as `on_0_tirith-iac-governance` in the dashboard and in +# every status key. Matches the step template's own name rather than describing the action, so a +# reader seeing the stage knows which template produced it. +POLICY_STEP_NAME = "tirith-iac-governance" +POLICY_STEP_TIMEOUT = 1800 + + +def policy_step(step_template_id, bundle_path): + """ + The pre-plan step entry, naming the bundle this run should evaluate. + + Sent in full on every run rather than relying on the copy stored on the workflow. core merges the + run's TerraformConfig over the workflow's (`workflowruns/__init__.py:1646`), and that merge is + shallow -- supplying `prePlanWfStepsConfig` replaces the whole list -- so the entry has to carry + its template id and timeout too, not just the path. + """ + return { + "name": POLICY_STEP_NAME, + "wfStepTemplateId": step_template_id or POLICY_STEP_TEMPLATE, + "timeout": POLICY_STEP_TIMEOUT, + "approval": False, + # Everything the step needs travels here. It reads nothing from the workflow's terraform + # configuration. + "wfStepInputData": { + "schemaType": "FORM_JSONSCHEMA", + "data": { + "bundlePath": bundle_path, + # Passed through so the step knows whether it may write the masked state to + # `artifacts/tfstate.json`. For a managed-state workflow that object *is* the live + # state, and a masked copy over it would be data loss. Always false here, because + # terraform_config below sets it false -- sent explicitly rather than relying on the + # step's default, so the intent is visible on every run. + "managedTerraformState": False, + }, + }, + } + + +def terraform_config(terraform_version, step_template_id): + """ + The workflow's stored configuration, carrying the policy step as a PRE-PLAN step. + + This is the whole mechanism, and it uses only primitives the platform already had. core splices + `prePlanWfStepsConfig` ahead of `generate-terraform-plan`, and a step exiting 12 tells the run + controller to complete the run successfully and skip everything after it. So the policy step runs, + exits 12, and the terraform plan never happens -- without core knowing anything about this feature. + + That is why the run's TerraformAction is `plan`: a dummy value, never acted on, chosen because it + is the action whose synthesis splices pre-plan steps in. + + `managedTerraformState` stays False -- a policy check writes no state, and it must not take the + managed-state backend override even on a workflow configured for one. + + Deliberately carries no "input kind". The step routes on which document is present in the + archive, because a stored kind cannot be trusted: a two-phase pipeline gates the plan and then + checks the state against the SAME workflow, whose identity derives from the repository and + workflow name. The workflow is created once, by whichever phase ran first, so the stored kind was + that phase's and the other phase fed its document to a provider that cannot read it. + + The `bundlePath` stored here is only a fallback. This configuration is written once, at workflow + creation -- `ensure_workflow` returns 409 for an existing workflow and updates nothing -- so it + cannot describe any particular run. Every run therefore sends its own `prePlanWfStepsConfig` in the + run body, which core merges over this one, naming that run's bundle. + """ + config = { + "terraformVersion": terraform_version or DEFAULT_TERRAFORM_VERSION, + "managedTerraformState": False, + "prePlanWfStepsConfig": [policy_step(step_template_id, ARCHIVE_DOCUMENT)], + } + return config + + +def write_output_json(path, payload): + if not path: + return + try: + with open(path, "w") as f: + json.dump(payload, f, indent=2) + except OSError as e: + log(f"WARNING: could not write {path}: {e}") + + +def _split_repo_url(repo_url): + """ + Return (sanitized_url, host) for a repo URL, or (None, None). + + **Strips userinfo.** `https://x-access-token:ghs_abc@github.com/acme/infra` is an ordinary value + for a CI checkout to hold, and GitLab's own `CI_REPOSITORY_URL` embeds a job token the same way. + Writing that into a file that ships inside the bundle would persist a credential in an artifact + that outlives the run. Sanitizing here rather than at the call site because it is the kind of thing + a later caller would forget. + + Handles scp syntax (`git@github.com:acme/infra.git`), which `urlsplit` reads as a path with no + host at all. + """ + if not repo_url: + return None, None + + text = repo_url.strip() + if "://" not in text and "@" in text: + # scp-style (`git@host:path`, and the `host/path` spelling a scheme-less CI variable produces). + # Rewritten to a URL shape so the host is recoverable, and so the userinfo is discarded rather + # than carried along. + _userinfo, _, remainder = text.rpartition("@") + host, separator, path = remainder.partition(":") + if not separator: + host, _, path = remainder.partition("/") + host = host.lower() + return (f"ssh://{host}/{path.lstrip('/')}", host) if host else (None, None) + + parts = urllib.parse.urlsplit(text) + try: + host = (parts.hostname or "").lower() or None + port = parts.port + except ValueError: + # An unparseable port raises rather than returning None. + host, port = None, None + + if not host: + # Fail closed. The input reached here *with* whatever userinfo it carried, and a URL we cannot + # parse is a URL we cannot sanitise -- returning it verbatim is how a token ends up in a file + # that ships inside the bundle and outlives the run. Both real-world shapes that land here + # carry credentials: `https://oauth2:${TOKEN}@${HOST}/x` with HOST unset renders an empty + # authority, and a scheme-less `user:token@host/path` parses its username as a scheme. Losing + # the URL from the metadata is a far cheaper failure than leaking the secret in it. + return None, None + + # hostname strips IPv6 brackets, so they have to go back or the authority is malformed. + literal = f"[{host}]" if ":" in host else host + authority = literal if port is None else f"{literal}:{port}" + return urllib.parse.urlunsplit((parts.scheme, authority, parts.path, parts.query, "")), host + + +def _repo_path(source_dir, declared=None): + """ + Where `code/` belongs inside the repository. Returns (path, how) with POSIX separators. + + This is the field an autofix consumer cannot do without: `--source-dir infra/prod` means `code/` + holds only that subtree, so `code/main.tf` has to be written back to `infra/prod/main.tf`. The + packing destroys that prefix -- members are named relative to the source directory -- so if it is + not recorded here it is unrecoverable. + + `""` means the repository root, and is deliberately not `None`: joining still works and it stays + distinguishable from "we could not tell", which is `None`. `how` is `"flag"` or `"git_root"`, so a + consumer about to write into someone's repository can tell a declared answer from an inferred one. + + Inference walks up for a `.git` entry rather than shelling out to git -- there is no git dependency + anywhere in this package, and a `.git` *file* (worktrees, submodules) counts. + """ + if declared is not None: + candidate = posixpath.normpath(declared.replace(os.sep, "/").strip("/")) + if candidate in (".", "/"): + return "", "flag" + # `..` here would have a consumer write outside the repository it thinks it is patching, which + # is the entire use of this field. Refuse it rather than record a path that escapes, and fall + # through to inference so the answer is merely absent rather than wrong. + if candidate.startswith("..") or posixpath.isabs(candidate): + log(f"WARNING: ignoring --repo-path {declared!r}: it must be a path inside the repository") + else: + return candidate, "flag" + if not source_dir: + return None, None + + try: + current = os.path.realpath(source_dir) + except OSError: + return None, None + + root = current + while True: + if os.path.exists(os.path.join(root, ".git")): + relative = os.path.relpath(current, root) + return ("" if relative == "." else relative.replace(os.sep, "/")), "git_root" + parent = os.path.dirname(root) + if parent == root: + return None, None + root = parent + + +def build_metadata(opts, redactions, absent_reason=None): + """ + The caller's half of `metadata.json`: what this bundle is. + + `archive.pack` fills in what it observes -- whether code was packed, under what prefix, and the + file counts -- so nothing here asserts a fact about the archive's contents. + + Two shapes of run have to produce an honest document. From CI, `--trigger-details-file` carries the + repository and commit. From a laptop there is no trigger payload at all (`{"type": "cli"}`), often + no `--sha` and no `--repo-url`; those fields are then `null` rather than omitted or invented, and + `origin.kind` says `local` as a positive statement instead of leaving CI to be inferred from an + absence. + + Field names are snake_case, matching every other JSON this tool *authors* -- the result document, + the manifest, and terraform's own plan.json sitting beside it. camelCase in this package appears + only where it mirrors the platform's wire API, which this file never touches. + """ + trigger = opts.trigger_details if isinstance(getattr(opts, "trigger_details", None), dict) else {} + trigger_type = trigger.get("type") or "cli" + url, host = _split_repo_url(getattr(opts, "repo_url", None) or trigger.get("repoHttpUrl")) + path, path_from = _repo_path(opts.source_dir, getattr(opts, "repo_path", None)) + + change_request = None + if trigger.get("prId"): + change_request = { + "id": str(trigger["prId"]), + "url": trigger.get("eventSource"), + "target_ref": trigger.get("baseRef"), + } + + return { + "schema_version": METADATA_SCHEMA_VERSION, + "generator": {"name": "tirith", "version": __version__}, + "created_at": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "input_kind": opts.input_kind, + "origin": { + # `cli` is what the CLI defaults the trigger type to when nothing supplied one, so it is + # the signal that no CI system was involved. + "kind": "local" if trigger_type == "cli" else "ci", + "trigger_type": trigger_type, + "ci_run_url": trigger.get("runUrl"), + }, + "repository": { + # Sniffed from the host, never from the CI provider: a GitHub Actions job can perfectly + # well check out a GitLab repository, so these are independent facts. + "provider": _KNOWN_VCS_HOSTS.get(host, "unknown"), + "host": host, + "url": url, + "ref": getattr(opts, "repo_ref", None) or trigger.get("ref"), + "commit": opts.sha or trigger.get("headSha"), + "change_request": change_request, + }, + "code": { + "repo_path": path, + "repo_path_from": path_from, + "absent_reason": absent_reason, + }, + "masking": { + # Named so a consumer can find masked values without hardcoding the sentinel, and knows + # not to feed this state to terraform. + "redactions": redactions, + "marker": redact.SENTINEL, + "documents_are_masked": True, + }, + "workflow": { + "org": opts.org, + "group": opts.workflow_group, + "id": opts.workflow_id, + "artifact_tag": opts.artifact_tag, + }, + } + + +def pack_documents(source_dir, plan, state, infracost, document_sources=(), metadata=None): + """ + Build the archive, dropping the source tree rather than failing if it is too large. + + Returns (bytes, manifest, source_skipped_reason) where the reason is None on the normal path. + + The source is packed by default, so an exclusion that does not fire -- a committed vendor + directory, a build output tree -- would otherwise turn a working policy check into a failed run. + That trade is the wrong way round: the verdict is what gates the merge, and the source is there + for the autofix system's benefit. So an oversized archive degrades to documents-only and says so, + loudly, rather than taking the gate down with it. + + Only when a source tree was actually requested. If we are already documents-only and still over + the limit, the *documents* are too big and there is nothing left to drop, so that stays fatal. + + A source directory that does not exist is a different thing entirely and must not degrade. It + raises `ArchiveError` too, so letting it reach the retry below reported a typo'd `--source-dir` as + "the tree was too large", dropped the code, and completed the run -- the check would pass having + silently evaluated no source at all. Caught here, where the distinction is still available. + """ + if source_dir and not os.path.isdir(source_dir): + raise CheckError(f"--source-dir does not exist: {source_dir}") + + try: + archive_bytes, manifest = archive.pack( + source_dir=source_dir, + plan=plan, + state=state, + infracost=infracost, + document_sources=document_sources, + metadata=metadata, + ) + return archive_bytes, manifest, None + except archive.ArchiveError as e: + if not source_dir: + raise + + reason = str(e) + log( + f"WARNING: {reason} Uploading the masked documents only, without the source. The policy " + f"check still runs, but the archive carries no code -- so anything reading it to generate " + f"fixes has nothing to work from. Point --source-dir at your terraform directory, or add " + f"the large paths to .gitignore." + ) + # The retry has to say *why* the code is missing, or a consumer cannot tell a deliberate + # documents-only run from a tree that was dropped for size. + retry_metadata = metadata + if metadata is not None: + retry_metadata = dict(metadata) + code = dict(metadata.get("code") or {}) + # Only overwrite a reason the caller did not already give. This path is reached solely + # when a source tree WAS requested and dropped -- `pack_documents` re-raises when there + # was none -- but stamping unconditionally would relabel a deliberate documents-only run + # as an oversize failure if the retry were ever reached another way. + if not code.get("absent_reason"): + code["absent_reason"] = "too_large" + retry_metadata["code"] = code + archive_bytes, manifest = archive.pack( + source_dir=None, plan=plan, state=state, infracost=infracost, metadata=retry_metadata + ) + return archive_bytes, manifest, reason + + +def upload_state_document(client, opts, state): + """ + Also publish the masked state as the workflow's `artifacts/tfstate.json`. + + That name is canonical rather than decorative: the managed-state backend writes it, state locking + keys on the literal basename, and the state-backends listing special-cases it. Putting the state + there is what makes it visible and downloadable in the platform's own State and artifacts views, + instead of being reachable only by unpacking the run's archive. + + It goes *in addition to* the copy inside the archive -- the step reads that one to publish + `TfStateCleaned`, and the two must not diverge. + + Best-effort: the check's verdict does not depend on it, so a failure warns rather than failing a + run whose policies evaluated perfectly well. + """ + if client.manages_terraform_state(opts.workflow_group, opts.workflow_id): + log( + "WARNING: not writing tfstate.json -- this workflow manages its own terraform state, and " + "that object is the live state. Overwriting it with a masked document would be data loss. " + "The state is still evaluated, and still in the run's archive." + ) + return + + try: + key = client.upload_file( + opts.workflow_group, + opts.workflow_id, + STATE_DOCUMENT_NAME, + None, + json.dumps(state).encode("utf-8"), + content_type=STATE_CONTENT_TYPE, + ) + except SGError as e: + log(f"WARNING: could not publish {STATE_DOCUMENT_NAME}: {e}") + return + + log( + f"Published the state document: {key} -- masked, so it reflects what was evaluated and " + f"cannot be used to run terraform." + ) + + +def run_check(opts): + """ + Execute the check. Returns the result document. + + Raises CheckError for anything that leaves the verdict unknown -- the caller maps that to a + non-zero exit regardless of --fail-on-error, because a run that produced no verdict must never + look like a pass. + """ + client = SGClient(opts.api_url, opts.org, opts.api_key, timeout=60) + + plan, state, infracost, redactions = prepare_documents( + opts.input_path, + opts.input_kind, + opts.state_path, + opts.infracost_path, + input_document=getattr(opts, "input_document", None), + ) + if redactions: + log(f"Masked {redactions} sensitive value(s) before upload") + + # Every path a document was read from, so the source walk cannot ship the unmasked original + # beside the masked copy. + # + # `plan_file` belongs here most of all, and was the omission that made this half a fix: + # --plan-file converts the BINARY plan in memory precisely so nothing unmasked touches the + # disk, but the binary plan itself is already on disk, and it embeds the prior state -- every + # attribute of every existing resource. The `tfplan` name patterns in DEFAULT_EXCLUDES only + # cover the spellings the README happens to use; `terraform plan -out=plan.out` is at least as + # common, and that file is the one thing here worth protecting most. + archive_bytes, manifest, source_skipped = pack_documents( + opts.source_dir, + plan, + state, + infracost, + document_sources=(opts.input_path, opts.state_path, opts.infracost_path, getattr(opts, "plan_file", None)), + metadata=build_metadata( + opts, + redactions, + absent_reason=None if opts.source_dir else "not_requested", + ), + ) + log( + f"Packed {manifest['files']} file(s) and {len(manifest['documents'])} document(s) " + f"into {manifest['bytes'] // 1024} KB" + ) + + try: + client.ensure_workflow_group(opts.workflow_group) + client.ensure_workflow( + opts.workflow_group, + opts.workflow_id, + f"Policy checks for {opts.workflow_id}", + terraform_config(opts.terraform_version, opts.step_template_id), + vcs_config=SGClient.vcs_config(getattr(opts, "repo_url", None), getattr(opts, "repo_ref", None)), + ) + + # A flat, fixed name at the artifact root, overwritten every run. The step finds it there + # because the run controller syncs that directory down before any step executes -- which is + # what removes the need for any run-creation field, and therefore for any api change at all. + bundle_name = ARCHIVE_NAME_TEMPLATE.format(sha=opts.sha[:7] if opts.sha else "latest", tag=opts.artifact_tag) + key = client.upload_file( + opts.workflow_group, + opts.workflow_id, + bundle_name, + None, + archive_bytes, + ) + log(f"Uploaded the project archive: {key}") + + if state is not None: + upload_state_document(client, opts, state) + + # The run names its own bundle. core merges this over the workflow's stored TerraformConfig, + # which is what makes the name per-run even though the workflow's copy was written once and + # never updated -- and therefore what lets the name carry the commit instead of being shared + # by every run of the workflow. + run_id, _data = client.create_run( + opts.workflow_group, + opts.workflow_id, + opts.trigger_details, + pre_plan_steps=[policy_step(opts.step_template_id, bundle_name)], + ) + except SGError as e: + raise CheckError(str(e)) + + # Quoted, the way client.py quotes the same three values on every API path it builds. This one is + # rendered into an `href` in the pull-request comment, so an unquoted value could put a space or a + # quote into a URL a reviewer clicks. The renderer escapes it as well; both, because neither alone + # is obviously sufficient at the point you are reading only one of them. + run_url = ( + f"{opts.dashboard_url.rstrip('/')}/orchestrator/orgs/{urllib.parse.quote(opts.org)}" + f"/wfgrps/{urllib.parse.quote(opts.workflow_group)}/wfs/{urllib.parse.quote(opts.workflow_id)}" + f"/wfruns/{urllib.parse.quote(str(run_id))}" + ) + log(f"Run created: {run_url}") + + # Written before polling so a timeout still leaves the run discoverable. + write_output_json(opts.output_json, {"status": "RUNNING", "wfrun_id": run_id, "wfrun_url": run_url}) + + try: + status, _run = client.wait_for_run( + opts.workflow_group, + opts.workflow_id, + run_id, + timeout=opts.timeout, + on_poll=lambda s: log(f"Run status: {s}"), + ) + except SGError as e: + raise CheckError(f"{e} (run: {run_url})") + + # The run facts are the source of truth -- they are what the dashboard renders. Fetched once: + # the document carries the verdict and the cost estimate, and it embeds the whole plan, so it + # is large enough that fetching it twice is worth avoiding. + # A read failure is held rather than raised straight away: an older step image publishes its + # verdict as an artifact instead, and that fallback below is still worth trying. What must not + # happen is a failed read falling through to an empty result set, which renders as "no policies + # in scope" -- a clean-looking exit for a run whose policies may well have failed. + facts_error = None + try: + facts = client.get_run_facts(opts.workflow_group, opts.workflow_id, run_id) + except SGError as e: + facts = {} + facts_error = e + + policy_results = facts.get("PolicyEvalResults") or {} + # PreApply is what the step writes for a check run; the bare key is the fallback for an older + # step image that only set that one. + cost_breakdown = facts.get("InfracostBreakdownPreApply") or facts.get("InfracostBreakdown") + + # The results artifact is only consulted when the facts come back empty, which means an older + # step image that still writes it. + legacy = None + if not policy_results: + legacy = client.get_results_artifact(opts.workflow_group, opts.workflow_id, f"{run_id}/tirith-results.json") + if legacy is not None: + policy_results = legacy + + # Only when NOTHING answered. `legacy is not None` means the artifact was read and was + # legitimately empty -- an older step image with no policies in scope -- which is a real + # no-policies result, not a failed read. + if facts_error is not None and legacy is None: + raise CheckError(f"The run completed but its results could not be read: {facts_error} (run: {run_url})") + + # The archive is deliberately retained. It is the source that produced these findings, and the + # autofix system reads it to generate fixes -- so deleting it here would remove the only copy of + # what was actually evaluated. + # + # One object per workflow, replaced on every run, so retention costs a bounded amount rather than + # growing per commit. It does land in the artifact prefix that is synced into every later run of + # the workflow -- unavoidable, because that sync is how the step receives it -- but the step + # deletes it from the volume after unpacking, so it does not travel onward from there. + # + # `client.delete_artifact` is kept for a retention sweep to use later. Note it currently points at + # a view that serves only GET and POST. + log(f"Retained the project archive for autofix: {key}") + + counts, _findings = report.summarize(policy_results) + verdict_value = report.verdict(counts, status) + + result = { + "status": status, + "verdict": verdict_value, + "counts": { + "passed": counts.get(report.PASS, 0), + "failed": counts.get(report.FAIL, 0), + "warned": counts.get(report.WARN, 0), + "approval_required": counts.get(report.APPROVAL_REQUIRED, 0), + "skipped": counts.get("SKIPPED", 0), + # Published so a consumer can tell "nothing failed" from "we could not read part of + # it". Without it an errored run reported failed: 0, which the action copies straight + # to its `failed` output. + "unknown": counts.get(report.UNKNOWN, 0), + }, + "headline": report.headline(counts, verdict_value), + "wfrun_id": run_id, + "wfrun_url": run_url, + "policy_results": policy_results or {}, + # Surfaced for a caller aggregating several units into one comment of their own. + "monthly_cost": (cost_breakdown or {}).get("totalMonthlyCost"), + # Where the evaluated source lives. The autofix system reads this to fetch what produced + # the findings; it is also recorded on the run itself as SGCustomWorkflowRunFacts, so a + # consumer holding only a run id can find it without seeing this document. + "archive_key": key, + # Whether that archive actually contains the source. Normally true, and false when the tree + # was too large and got dropped so the check could still run. A consumer must not assume: + # "no code in the bundle" and "no code was wanted" need to be distinguishable. + # Derived from what the archive actually holds, not from what was asked for: a tree whose + # every file was excluded packs nothing, and this must not then claim otherwise while + # metadata.json says `present: false`. + "source_packed": bool(manifest.get("files")), + "source_skipped_reason": source_skipped, + } + + write_output_json(opts.output_json, result) + + if opts.output_markdown: + body = report.render_markdown( + policy_results, + status, + run_url, + marker=opts.comment_marker, + limit=opts.markdown_limit, + cost_breakdown=cost_breakdown, + commit=opts.sha, + ) + try: + with open(opts.output_markdown, "w") as f: + f.write(body) + except OSError as e: + log(f"WARNING: could not write {opts.output_markdown}: {e}") + + log(result["headline"]) + return result diff --git a/src/tirith/platform/cli.py b/src/tirith/platform/cli.py new file mode 100644 index 00000000..c4070b3e --- /dev/null +++ b/src/tirith/platform/cli.py @@ -0,0 +1,283 @@ +""" +`tirith platform ...` -- run policy checks against a StackGuardian organization. + +Flag and environment names follow sg-cli (SG_API_TOKEN, SG_BASE_URL, SG_ORG, SG_DASHBOARD_URL) so +someone who knows one tool knows the other. `--region` names both URLs at once; see regions.py for +the precedence between it, the explicit flags and the environment. +""" + +import argparse +import json +import os +import re +import sys + +from ..status import ExitStatus +from . import discover, regions +from .check import DEFAULT_WORKFLOW_GROUP, INPUT_KINDS, CheckError, log, run_check + +# `Id` is a DRF SlugField on the platform, and the value is interpolated into every API path. +WORKFLOW_ID_PATTERN = re.compile(r"^[A-Za-z0-9_-]{1,100}$") + + +def _resolve_api_key(value): + """ + Resolve the API key, preferring the environment. + + A key on argv is visible in `ps` for the lifetime of the process, so `-` reads it from stdin + and $SG_API_TOKEN is the documented default. + """ + if value == "-": + return sys.stdin.readline().strip() + return value or os.environ.get("SG_API_TOKEN", "") + + +def _load_trigger_details(opts): + if opts.trigger_details_json: + source, raw = "--trigger-details-json", opts.trigger_details_json + elif opts.trigger_details_file: + source = f"--trigger-details-file {opts.trigger_details_file}" + try: + with open(opts.trigger_details_file) as f: + raw = f.read() + except OSError as e: + raise CheckError(f"Could not read {opts.trigger_details_file}: {e}") + else: + return {"type": "cli"} + + try: + details = json.loads(raw) + except json.JSONDecodeError as e: + raise CheckError(f"{source} is not valid JSON: {e}") + if not isinstance(details, dict): + raise CheckError(f"{source} must be a JSON object") + details.setdefault("type", "cli") + return details + + +def build_parser(): + parser = argparse.ArgumentParser( + prog="tirith platform", + description="Run StackGuardian policy checks from a CI pipeline or a laptop.", + ) + sub = parser.add_subparsers(dest="subcommand") + + check = sub.add_parser( + "check", + help="Evaluate the organization's policies against a document and report the verdict.", + description=( + "Masks the document, packs it with the terraform source into an archive, uploads it, " + "runs the policies on StackGuardian and reports the verdict." + ), + ) + + identity = check.add_argument_group("identity") + identity.add_argument( + "--api-key", default=None, help="API key, or '-' to read it from stdin. Default: $SG_API_TOKEN" + ) + identity.add_argument("--org", default=None, help="Organization name. Default: $SG_ORG") + identity.add_argument( + "--region", + default=None, + choices=regions.REGION_IDS, + help=( + f"StackGuardian region, setting both URLs at once. " f"Default: $SG_REGION or {regions.DEFAULT_REGION_ID}." + ), + ) + identity.add_argument( + "--api-url", + default=None, + help=( + "API base URL, with or without /api/v1. Overrides --region; needed only for a " + "self-hosted install or a dedicated host. Default: $SG_BASE_URL" + ), + ) + identity.add_argument( + "--dashboard-url", + default=None, + help="Dashboard base URL, used to build run links. Inferred from --api-url when it names a known region.", + ) + + workflow = check.add_argument_group("workflow") + workflow.add_argument( + "--workflow-id", + required=True, + help="Slug identifying the workflow. Created if absent. Letters, digits, '-' and '_' only.", + ) + workflow.add_argument("--workflow-group", default=DEFAULT_WORKFLOW_GROUP, help="Workflow group. Created if absent.") + workflow.add_argument("--terraform-version", default=None, help="Stored on the workflow at creation.") + workflow.add_argument( + "--repo-url", + default=None, + help="Source repository URL, recorded on the workflow at creation so it links back to the code.", + ) + workflow.add_argument("--repo-ref", default=None, help="Branch, tag or commit, recorded alongside --repo-url.") + workflow.add_argument( + "--repo-path", + default=None, + help=( + "Path of --source-dir within the repository, recorded in the bundle's metadata.json so a " + "consumer knows where code/ belongs. Inferred from the enclosing git checkout if omitted." + ), + ) + workflow.add_argument( + "--step-template-id", + default=None, + help="Override the policy-evaluation step template. Omit to use the platform's own default.", + ) + + inputs = check.add_argument_group("inputs") + inputs.add_argument( + "--input-path", + default=None, + help=( + "Document to evaluate. Defaults to whichever of " + f"{' or '.join(discover.PLAN_FILENAMES)} is in --source-dir." + ), + ) + inputs.add_argument( + "--plan-file", + default=None, + help=( + "Binary plan from `terraform plan -out=`. Rendered with `show -json` in memory, so no " + "unmasked plan JSON is written to disk. Use --input-path if you already have the JSON." + ), + ) + inputs.add_argument( + "--terraform-bin", + default=None, + help="terraform/tofu binary for --plan-file. Auto-detected, preferring the real binary over a CI wrapper.", + ) + inputs.add_argument("--input-kind", default="terraform_plan", choices=INPUT_KINDS) + inputs.add_argument("--state-path", default=None, help="Optional terraform state, masked before upload.") + inputs.add_argument("--infracost-path", default=None, help="Optional `infracost breakdown --format json`.") + inputs.add_argument("--source-dir", default=".", help="Terraform source to pack alongside the documents.") + inputs.add_argument( + "--no-source", + action="store_true", + help="Send only the documents. Discovery still looks in --source-dir (or .) for the plan.", + ) + + run = check.add_argument_group("run") + run.add_argument("--sha", default=None, help="Commit SHA, used to namespace the uploaded archive.") + run.add_argument( + "--artifact-tag", + default="default", + help=( + "Namespaces the archive within a commit. Needed only when one workflow evaluates the same " + "commit more than once -- a plan phase and a state phase, or matrix legs sharing a workflow." + ), + ) + run.add_argument("--trigger-details-json", default=None, help="JSON object describing what triggered this run.") + run.add_argument("--trigger-details-file", default=None, help="File containing that JSON object.") + run.add_argument("--timeout", type=int, default=1800, help="Seconds to wait for the run. Default: 1800") + + output = check.add_argument_group("output") + output.add_argument("--output-json", default=None, help="Write the result document here.") + output.add_argument("--output-markdown", default=None, help="Write a markdown report here.") + output.add_argument("--comment-marker", default=None, help="Opaque first line of the markdown, for stickiness.") + output.add_argument("--markdown-limit", type=int, default=60000, help="Truncate the markdown to this length.") + output.add_argument( + "--fail-on-error", + action="store_true", + help=( + "Exit non-zero when a policy fails. An unreachable platform or a run that produced no " + "verdict always exits non-zero regardless of this flag." + ), + ) + + return parser + + +def main(argv): + parser = build_parser() + opts = parser.parse_args(argv[1:]) + + if opts.subcommand != "check": + parser.print_help() + return ExitStatus.SUCCESS + + opts.api_key = _resolve_api_key(opts.api_key) + opts.org = opts.org or os.environ.get("SG_ORG", "") + try: + opts.api_url, opts.dashboard_url, url_warnings = regions.resolve( + region_id=opts.region, + api_url=opts.api_url, + dashboard_url=opts.dashboard_url, + env=os.environ, + ) + except ValueError as e: + log(f"ERROR: {e}") + return ExitStatus.ERROR + for warning in url_warnings: + log(f"WARNING: {warning}") + opts.source_dir = None if opts.no_source else opts.source_dir + + missing = [name for name, value in (("--api-key", opts.api_key), ("--org", opts.org)) if not value] + if missing: + log(f"ERROR: missing required {' and '.join(missing)}") + return ExitStatus.ERROR + + if not WORKFLOW_ID_PATTERN.match(opts.workflow_id): + # Checked before any HTTP call: the value goes straight into every API path, and the + # platform's own field is a slug, so a `/` yields a malformed URL rather than a clear error. + suggestion = re.sub(r"[^A-Za-z0-9_-]+", "-", opts.workflow_id).strip("-").lower()[:100] + log(f"ERROR: --workflow-id '{opts.workflow_id}' is not a valid slug. Try '{suggestion}'.") + return ExitStatus.ERROR + + opts.input_document = None + if opts.plan_file: + if opts.input_path: + log("ERROR: --plan-file and --input-path cannot be combined; they name the same document") + return ExitStatus.ERROR + try: + opts.input_document = discover.terraform_show_json( + opts.plan_file, workdir=opts.source_dir, binary=opts.terraform_bin + ) + except discover.DiscoveryError as e: + log(f"ERROR: {e}") + return ExitStatus.ERROR + log(f"Rendered {opts.plan_file} with `terraform show -json`") + elif not opts.input_path and not opts.state_path: + # Nothing was named, so look in the conventional place. This is what lets a caller run with + # no configuration at all. + try: + opts.input_path = discover.discover_input(opts.source_dir) + except discover.DiscoveryError as e: + log(f"ERROR: {e}") + return ExitStatus.ERROR + log(f"Using {opts.input_path}") + + if opts.api_key.startswith("sgu_"): + log( + "WARNING: sgu_ tokens are non-functional for SSO-group-only users and inherit only " + "direct permissions for hybrid SSO users. Prefer an organization (sgo_) token." + ) + + try: + opts.trigger_details = _load_trigger_details(opts) + result = run_check(opts) + except CheckError as e: + # Fails closed: a run that produced no verdict must never look like a pass, whatever + # --fail-on-error says. + log(f"ERROR: {e}") + return ExitStatus.ERROR + except KeyboardInterrupt: + log("Interrupted") + return ExitStatus.ERROR_CTRL_C + + verdict = result["verdict"] + if verdict == "errored": + # Fails closed regardless of --fail-on-error: the flag governs policy verdicts, not tool + # health, and a run that produced no verdict must never look like a pass. + log("The run did not produce a verdict") + return ExitStatus.ERROR + if verdict == "failed" and opts.fail_on_error: + return ExitStatus.ERROR_POLICY_FAILED + if verdict == "failed": + log("Policies failed, but --fail-on-error was not set") + # A policy asking for approval warns rather than gating -- see report.verdict for why. + if result.get("counts", {}).get("approval_required"): + log("Some policies ask for approval; reported as a warning, which does not block") + + return ExitStatus.SUCCESS diff --git a/src/tirith/platform/client.py b/src/tirith/platform/client.py new file mode 100644 index 00000000..95574975 --- /dev/null +++ b/src/tirith/platform/client.py @@ -0,0 +1,502 @@ +""" +StackGuardian API client. + +stdlib only -- urllib rather than requests -- so this adds no dependency to a package that has +three, and a CI runner needs nothing installed beyond tirith itself. + + POST /orgs//wfgrps/ create the workflow group + POST /orgs//wfgrps//wfs/ create the workflow + GET /orgs//wfgrps//wfs//file_upload_url/ presigned PUT (5 min) + key + POST /orgs//wfgrps//wfs//wfruns/ create the run + GET /orgs//wfgrps//wfs//wfruns// poll + GET /orgs//wfgrps//wfs//artifacts// fetch the results artifact + GET .../wfruns//wfrunfacts// fallback -> PolicyEvalResults +""" + +import gzip +import json +import time +import urllib.error +import urllib.parse +import urllib.request + +from . import regions + +# Signed into the upload URL by the platform, so the PUT must send the same value. +# The bundle is PUT to a URL the platform signs for application/json regardless of filename, and S3 +# validates the signature against the header the client sends -- not against the body. So the header +# has to be the signed one even though the body is gzip. Sending application/gzip earns a +# SignatureDoesNotMatch; the stored object is merely labelled wrongly, which nothing reads. +ARCHIVE_CONTENT_TYPE = "application/json" + +# The bundle's name in the workflow's artifact directory, per commit and tag. +# +# Namespaced rather than fixed because a fixed name is shared by every run of the workflow, and two +# runs overlapping -- two pull requests, which is routine, since the action derives one workflow id per +# repository -- would leave one run evaluating the other's code and reporting the verdict as its own. +# Silent, and wrong in the direction that gates a merge. A per-commit name cannot collide, so the race +# does not exist rather than being detected after the fact. +# +# The cost is growth: the artifact directory is synced *down* into every later run of the workflow, the +# up-sync carries no --delete, and api exposes no artifact DELETE, so bundles accumulate and every run +# pays to download all of them. Accepted deliberately -- correctness over transfer cost -- and the +# reason `delete_artifact` below is kept for a retention sweep to use. +# +# Flat, because a nested key cannot be deleted correctly: the authorizer's greedy +# converter swallows it, so `DELETE .../artifacts///` matches the workflow-group delete and +# is checked against the wrong permission entirely. +# +# The name is constrained more than it looks. The down-sync excludes `sg.*`, `*__sg.*`, `*pci_*`, +# `*_thrifty_*`, `*_gdpr_*`, `*compliance_raw*` and the other compliance globs, so a name matching any +# of those would be dropped silently and never reach the container. It also must not be +# `tfstate.json`, which at the artifact root is a managed-state workflow's live state. +ARCHIVE_NAME_TEMPLATE = "tirith-bundle-{sha}-{tag}.tar.gz" + +# What the workflow stores as a fallback, and what the step falls back to if a run names nothing. +ARCHIVE_DOCUMENT = "tirith-bundle.tar.gz" + +# Terminal run states. QUEUED/PENDING/RUNNING are transient; a run can sit in QUEUED for a long +# while behind the per-workflow concurrency gate, which is why the caller logs each poll. +# +# APPROVAL_REQUIRED is terminal *for polling purposes*: it is a resting state, reached when a +# policy's onFail is APPROVAL_REQUIRED, and nothing further happens without a human. Treating it as +# transient would spin until the timeout and then report a tool failure for what is actually a +# completed evaluation. sg-cli treats it the same way. +TERMINAL_STATUSES = ("COMPLETED", "ERRORED", "CANCELLED", "APPROVAL_REQUIRED") + +RETRYABLE_STATUS = (408, 429, 500, 502, 503, 504) + + +class SGError(Exception): + """An API call failed in a way the caller cannot recover from.""" + + +def _extract_signed_url(payload): + """ + Pull the presigned URL out of an upload-url response. + + The shape varies by endpoint and deployment: the tfstate/file upload endpoints return the URL + as a bare string in `msg`, while the newer template-artifact endpoints nest it under + `data.signedUrl`. Accept either rather than depending on one. + """ + if not isinstance(payload, dict): + return None + + for container_key in ("data", "msg"): + container = payload.get(container_key) + if isinstance(container, str) and container.startswith("http"): + return container + if isinstance(container, dict): + for url_key in ("signedUrl", "signed_url", "url"): + candidate = container.get(url_key) + if isinstance(candidate, str) and candidate.startswith("http"): + return candidate + return None + + +class SGClient: + def __init__(self, api_url, org, api_key, user_agent="tirith-action", timeout=60): + # Accepts a base with or without /api/v1, so a SG_BASE_URL exported for sg-cli works here. + self.api_url = regions.normalize_api_url(api_url) or regions.normalize_api_url( + regions.by_id(regions.DEFAULT_REGION_ID).api_base + ) + self.org = org + self.api_key = api_key + self.user_agent = user_agent + self.timeout = timeout + + # -- plumbing ------------------------------------------------------------------------------ + + def _request(self, method, path, body=None, retries=4): + url = f"{self.api_url}/orgs/{urllib.parse.quote(self.org)}{path}" + data = json.dumps(body).encode() if body is not None else None + + last_error = None + for attempt in range(retries + 1): + request = urllib.request.Request(url, data=data, method=method) + # SG's documented scheme. Must be an sgo_ (org) token: sgu_ tokens are non-functional + # for SSO-group-only users and inherit only direct permissions for hybrid SSO users, + # which surfaces as a confusing 403. + request.add_header("Authorization", f"apikey {self.api_key}") + request.add_header("Content-Type", "application/json") + request.add_header("X-SG-Client", self.user_agent) + + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + raw = response.read() + return response.status, (json.loads(raw) if raw else {}) + except urllib.error.HTTPError as e: + raw = e.read() + try: + payload = json.loads(raw) if raw else {} + except json.JSONDecodeError: + payload = {"msg": raw.decode("utf-8", "replace")[:500]} + + if e.code in RETRYABLE_STATUS and attempt < retries: + last_error = f"HTTP {e.code}: {payload.get('msg', '')}" + time.sleep(min(2**attempt, 8)) + continue + return e.code, payload + except (urllib.error.URLError, TimeoutError) as e: + # Never treat a network failure as a pass -- the caller maps this to a red check. + last_error = str(e) + if attempt < retries: + time.sleep(min(2**attempt, 8)) + continue + raise SGError(f"Could not reach StackGuardian at {self.api_url}: {last_error}") + + raise SGError(f"StackGuardian request failed after {retries + 1} attempts: {last_error}") + + # -- resources ----------------------------------------------------------------------------- + + def ensure_workflow_group(self, name): + """ + Create the workflow group if absent. + + Needed because `createIfNotExists` on run creation auto-creates the *workflow*, not the + group -- core's own error for a missing group reads "Workflow Group does not exist and + cannot be created". A 409 means someone else already made it, which is success here. + """ + status, payload = self._request( + "POST", + "/wfgrps/", + {"ResourceName": name, "Description": "Created by tirith", "Tags": ["sg-created"]}, + ) + if status in (200, 201, 409): + return status + raise SGError(f"Could not create workflow group '{name}' (HTTP {status}): {payload.get('msg')}") + + @staticmethod + def vcs_config(repo_url, repo_ref=None): + """ + Build the workflow's VCSConfig from a repo URL, recording where the code came from. + + `GIT_OTHER` -- singular, the wire value behind the UI's "Git Others" -- is the + connector-less provider. With `isPrivate: false` it needs no auth at all, and it skips the + GitHub repo-id extraction that rejects anything it cannot parse as an owner/name pair. + + This is display metadata, set on the *workflow* so it shows a repo link instead of a + "configure" prompt. It is not a source of code: every run sends `VCSConfig: {}` to suppress + the checkout (see `create_run`). Keeping the two apart is deliberate -- the workflow records + where the code came from, the run declines to fetch it. + + It cannot be made inert by shape alone. Dropping `useMarketplaceTemplate`, which is what + actually arms the runner's clone, is rejected by api: `IACVCSConfig` declares it + `BooleanField(required=True)` (`serializers/commons.py:141`). Send `iacVCSConfig` at all and + the key comes with it. + """ + if not repo_url: + return None + config = {"isPrivate": False, "repo": repo_url} + if repo_ref: + config["ref"] = repo_ref + return { + "iacVCSConfig": { + "useMarketplaceTemplate": False, + "customSource": {"sourceConfigDestKind": "GIT_OTHER", "config": config}, + } + } + + def ensure_workflow(self, wfgrp, workflow_id, description, terraform_config, vcs_config=None): + """ + Create the workflow if absent, keyed on `Id`. + + `Id` is the stable slug identity and what goes in the URL; `ResourceName` is a display name + and is not unique. Both are set to the same string so there is one name to reason about. + Note `Id` is a DRF SlugField, so it cannot contain dots. + + The workflow is `TERRAFORM`, not `CUSTOM`. For a terraform workflow core synthesises the + steps from the stored TerraformConfig plus the per-run TerraformAction and *ignores* any + WfStepsConfig in the request -- so the step configuration has to live here, once, rather + than being sent on every run. It also means the run renders as a real terraform run in the + dashboard rather than as opaque custom steps. + + `vcs_config` is set on creation only -- a 409 means the workflow already exists and nothing + is updated, so a workflow created before this existed keeps its blank repo field. + """ + body = { + "Id": workflow_id, + "ResourceName": workflow_id, + "Description": description, + "Tags": ["sg-created", "tirith"], + "WfType": "TERRAFORM", + "TerraformConfig": terraform_config, + } + if vcs_config: + body["VCSConfig"] = vcs_config + + status, payload = self._request("POST", f"/wfgrps/{urllib.parse.quote(wfgrp)}/wfs/", body) + if status in (200, 201, 409): + return status + raise SGError(f"Could not create workflow '{workflow_id}' (HTTP {status}): {payload.get('msg')}") + + def manages_terraform_state(self, wfgrp, workflow_id): + """ + Whether the workflow keeps its terraform state on the platform. + + Consulted before writing `artifacts/tfstate.json`, because for a managed-state workflow that + object *is* the live state: the step's backend writes it, state locking keys on the literal + name, and the state-backends view lists it. Overwriting it with a masked document would be + data loss, so this is a hard gate rather than a warning. + + Unreadable answers as True -- the safe direction. Not being able to tell whether an object is + live state is not a reason to overwrite it. + """ + status, payload = self._request("GET", f"/wfgrps/{urllib.parse.quote(wfgrp)}/wfs/{workflow_id}/") + if status != 200: + return True + body = payload.get("msg") or payload.get("data") or {} + if not isinstance(body, dict) or "TerraformConfig" not in body: + # A 200 that carries no TerraformConfig is still an answer we cannot read. Absent is not + # the same as false. + return True + return bool((body.get("TerraformConfig") or {}).get("managedTerraformState")) + + # `content` rather than `payload`: the response variable below is already called payload, and + # shadowing it sent the JSON response body to S3 in place of the file. + def upload_file(self, wfgrp, workflow_id, filename, folder, content, content_type=ARCHIVE_CONTENT_TYPE): + r""" + Upload one object into the workflow's artifact prefix via a presigned PUT, returning its key. + + The returned key is informational -- a log line, and something to quote in a bug report. It is + deliberately not load-bearing: nothing passes it back on the run, and the step finds the bundle + by *basename* inside the artifact directory the run controller syncs down for it. That is why + an api which returns no key at all is fine here. It comes from the response rather than being + rebuilt because the layout is runner-aware (a private runner's own S3 bucket or Azure container + rather than the shared bucket), so a client-side guess would be wrong for exactly the customers + who are hardest to debug. + + `folder` is optional and must be a flat token -- the endpoint rejects `/`, `\\` and `..` to + prevent path traversal. Omitting it puts the object at the artifacts root, which is what both + callers want: the archive because a nested key cannot be deleted correctly, and the state + document because `artifacts/tfstate.json` is the canonical location the platform reads. + """ + # No `contentType` parameter. The endpoint signs application/json regardless, and asking it to + # sign anything else needs an api change this feature deliberately does not make -- so the PUT + # below sends application/json to match the signature, and the bundle is merely labelled + # wrongly in storage. Nothing reads that label. + params = {"filename": filename} + if folder: + # Only when set. urlencode stringifies None to the literal "None", and the endpoint + # treats any non-empty value as a subfolder -- so passing it unconditionally produced a + # real `None/` directory in S3, and the archive then sat at a nested key that the + # post-run delete could not address. + params["folder"] = folder + query = urllib.parse.urlencode(params) + status, payload = self._request( + "GET", f"/wfgrps/{urllib.parse.quote(wfgrp)}/wfs/{workflow_id}/file_upload_url/?{query}" + ) + if status != 200: + raise SGError(f"Could not get an upload URL for {filename} (HTTP {status}): {payload.get('msg')}") + + # Informational only, and optional. It used to be required, because the caller had to pass the + # key back as a run field -- and an api that did not return it produced a run pointing at + # nothing. Nothing passes the key anywhere now: the step finds the bundle by name in the + # artifacts directory. So an api that does not return a key is fine, and this stays a label for + # the log line rather than a hard requirement. + key = (payload.get("data") or {}).get("key") or f"{filename} (key not reported)" + signed_url = _extract_signed_url(payload) + if not signed_url: + raise SGError(f"No signed URL in the upload response for {filename}: {payload}") + + # Must match the content type the URL was signed with, or S3 rejects it as a signature + # mismatch. + put = urllib.request.Request(signed_url, data=content, method="PUT") + put.add_header("Content-Type", content_type) + try: + with urllib.request.urlopen(put, timeout=self.timeout) as response: + if response.status not in (200, 204): + raise SGError(f"Upload of {filename} returned HTTP {response.status}") + except urllib.error.HTTPError as e: + # The signed URL is valid for 5 minutes; an expiry shows up here as a 403. + raise SGError(f"Upload of {filename} failed (HTTP {e.code}): {e.read()[:300]!r}") + except (urllib.error.URLError, TimeoutError) as e: + raise SGError(f"Upload of {filename} failed: {e}") + + return key + + def create_run(self, wfgrp, workflow_id, trigger_details, pre_plan_steps=None, action="plan"): + """ + Create one workflow run. Every invocation makes a new run. + + Deliberately carries no WfStepsConfig: core ignores that for TERRAFORM workflows, synthesising + the steps from TerraformConfig and TerraformAction instead. `TerraformConfig` is the field it + *does* honour per run -- core merges the run's over the workflow's + (`workflowruns/__init__.py:1646`) -- so that is how each run names its own bundle. + + The merge is shallow, so `prePlanWfStepsConfig` replaces the workflow's list wholesale and the + caller must send the complete step entry. Keys it does not send, `terraformVersion` and + `managedTerraformState`, still come from the workflow. + + Note what is *not* here: any archive field. The bundle reaches the step through the workflow's + artifact directory, which the run controller syncs down before any step runs, and the step is + told which one to read via that step's `wfStepInputData`. So api needs no new serializer field + and no new response key -- `TerraformConfig` is already declared on WorkflowRunSerializer. + + `terraformProjectZip` was the previous carrier and is gone. It worked, but it cost a declared + field in api: DRF drops undeclared keys, so without that change a run came back 201 having + silently discarded the reference and would have evaluated a VCS checkout instead of the + uploaded code. + + A context tag was the other obvious-looking option and is the wrong tool: run context tags are + indexed into global search, so an internal storage key would surface in customers' tag + typeaheads and could be enumerated by filtering on it. + + `VCSConfig: {}` suppresses the checkout for this run. The workflow keeps its own VCSConfig so + the dashboard still shows which repository the runs came from, but core resolves the run's + copy as `data.get("VCSConfig", wfDetails.get("VCSConfig", {}))` + (`workflowruns/__init__.py:1770`) -- a *present* empty value beats the workflow's, while + omitting the key inherits it. The runner then clones only when + `vcsConfig.iacVCSConfig` carries a `useMarketplaceTemplate` key (`external.py:2484`), so an + empty config skips git entirely. + + Sending it matters for two reasons. A private repository has no credentials here -- the + checkout died with "could not read Password for 'https://None@github.com'" before the step + ran -- and on a public one the clone quietly placed the *unmasked* source in the workspace, + which then reached S3 inside the run snapshot. The bundle is the only source this feature + wants on the platform. + """ + body = { + "TerraformAction": {"action": action}, + "TriggerDetails": trigger_details, + "VCSConfig": {}, + } + if pre_plan_steps: + body["TerraformConfig"] = {"prePlanWfStepsConfig": pre_plan_steps} + status, payload = self._request("POST", f"/wfgrps/{urllib.parse.quote(wfgrp)}/wfs/{workflow_id}/wfruns/", body) + if status not in (200, 201): + raise SGError(f"Could not create the workflow run (HTTP {status}): {payload.get('msg')}") + + data = payload.get("data") or {} + run_name = data.get("ResourceName") + if not run_name: + raise SGError(f"No ResourceName in the run-creation response: {payload}") + + return run_name, data + + def get_run(self, wfgrp, workflow_id, run_id): + status, payload = self._request( + "GET", f"/wfgrps/{urllib.parse.quote(wfgrp)}/wfs/{workflow_id}/wfruns/{run_id}/" + ) + if status != 200: + raise SGError(f"Could not read run {run_id} (HTTP {status}): {payload.get('msg')}") + # This endpoint returns the run object under "msg" rather than "data". + return payload.get("msg") or payload.get("data") or {} + + def wait_for_run(self, wfgrp, workflow_id, run_id, timeout=1800, interval=10, on_poll=None): + """ + Poll until the run reaches a terminal state. + + A timeout is a failure, never a pass: the caller maps it to a red check. `on_poll` exists + so the caller can log each status -- a run stuck in QUEUED behind another run on the same + workflow looks identical to a hung run otherwise. + """ + deadline = time.time() + timeout + last_status = None + + while time.time() < deadline: + run = self.get_run(wfgrp, workflow_id, run_id) + status = run.get("LatestStatus") + if status != last_status and on_poll: + on_poll(status) + last_status = status + + if status in TERMINAL_STATUSES: + return status, run + time.sleep(interval) + + raise SGError( + f"Run {run_id} did not finish within {timeout}s (last status: {last_status}). " + f"Runs on one workflow serialize, so it may be queued behind another run." + ) + + def get_results_artifact(self, wfgrp, workflow_id, artifact_path): + """ + Read the results artifact the tirith step used to publish next to the inputs. + + Kept only so a newer CLI still reads results from an older step image. Current step images + do not write this file: it carried exactly the PolicyEvalResults that the run facts already + hold, and it existed only because the facts endpoint used to answer "does not exist" for + every run. That was a key mismatch in the run controller, not a missing record. + + Returns None -- not {} -- when absent, so the caller can tell "no such artifact, go ask the + facts endpoint" from "the artifact exists and no policies matched". + """ + status, payload = self._request( + "GET", + f"/wfgrps/{urllib.parse.quote(wfgrp)}/wfs/{workflow_id}/artifacts/{artifact_path}/", + ) + if status != 200: + return None + + # This endpoint returns the artifact body directly rather than an envelope. + if isinstance(payload, dict) and "PolicyEvalResults" in payload: + return payload.get("PolicyEvalResults") or {} + return None + + def get_run_facts(self, wfgrp, workflow_id, run_id): + """ + Fetch the whole run-facts document. + + One call, because the document carries everything the caller reports on -- + PolicyEvalResults, the cost breakdown, the plan -- and it embeds the full plan, so it is + large enough that fetching it twice is worth avoiding. + + The endpoint hands back a presigned GET rather than the payload inline, for the same reason. + + Raises SGError when the facts could not be *read*, and returns {} only when they were read + and were empty. Collapsing both into {} made an unreadable run -- a 403 on the endpoint, a + failed presigned GET -- indistinguishable from a run with no policies in scope, so a run + whose policies had actually failed reported "no policies in scope" and exited 0. + """ + status, payload = self._request( + "GET", + f"/wfgrps/{urllib.parse.quote(wfgrp)}/wfs/{workflow_id}/wfruns/{run_id}/wfrunfacts/default/", + ) + if status == 404: + # Absent, not unreadable. A run that never produced a facts document answers this way, + # and that is a legitimate empty result -- treating it as a read failure would turn + # healthy runs red, which is the opposite of the mistake being fixed. + return {} + if status != 200: + raise SGError(f"Could not read the run facts for {run_id} (HTTP {status}): {payload.get('msg')}") + + body = payload.get("msg") or payload.get("data") or {} + if isinstance(body, dict) and body.get("PolicyEvalResults"): + return body + + # Via the shared helper: this endpoint returns `signed_url`, not `signedUrl`. Reading only + # the camelCase spelling meant this always fell through to {} -- which went unnoticed for as + # long as the results artifact was covering for it. + signed_url = _extract_signed_url(payload) + if not signed_url: + # A 200 carrying neither the facts inline nor a URL to them: the run genuinely has no + # facts document, which is what an empty result set looks like. + return {} + + try: + with urllib.request.urlopen(signed_url, timeout=self.timeout) as response: + raw = response.read() + if response.info().get("Content-Encoding") == "gzip" or raw[:2] == b"\x1f\x8b": + raw = gzip.decompress(raw) + return json.loads(raw) or {} + except Exception as e: + raise SGError(f"Could not fetch the run facts document for {run_id}: {e}") + + def get_policy_results(self, wfgrp, workflow_id, run_id): + """Read PolicyEvalResults from the run facts. This is the primary source of the verdict.""" + return self.get_run_facts(wfgrp, workflow_id, run_id).get("PolicyEvalResults") or {} + + def delete_artifact(self, wfgrp, workflow_id, artifact_name): + """ + Delete one artifact. Best-effort: returns True on success, False otherwise. + + `artifact_name` must be a single path segment. A nested name is swallowed by the greedy + converter in the authorizer and matches `DELETE .../wfgrps//` -- the + workflow-group delete -- so it would be checked against entirely the wrong permission. + """ + status, _payload = self._request( + "DELETE", + f"/wfgrps/{urllib.parse.quote(wfgrp)}/wfs/{workflow_id}/artifacts/{artifact_name}/", + ) + return status in (200, 204, 404) diff --git a/src/tirith/platform/discover.py b/src/tirith/platform/discover.py new file mode 100644 index 00000000..61f9384c --- /dev/null +++ b/src/tirith/platform/discover.py @@ -0,0 +1,131 @@ +""" +Find the document to evaluate without being told where it is. + +Exists so a caller with a plan in the conventional place needs no configuration at all. It lives +here rather than in the GitHub Action so GitLab, Jenkins and a local shell get the same behaviour. + +`terraform show -json` is also run from here, so a caller never has to write an unmasked plan to +disk at all -- see `terraform_show_json` for why resolving the right binary matters. +""" + +import json +import os +import shutil +import subprocess + +# Tried in order. Two names, not a glob: a glob over *.json would sweep up an infracost breakdown or +# a package manifest and evaluate it as a plan. +PLAN_FILENAMES = ("plan.json", "tfplan.json") + + +class DiscoveryError(Exception): + """No document could be resolved. Always fails closed.""" + + +def discover_input(source_dir): + """ + Find the plan document in `source_dir`, by convention. + + Two matches is an error rather than "first one wins". Silently evaluating the wrong document + would report a verdict about infrastructure the caller did not ask about, and look like a pass. + """ + directory = source_dir or "." + found = [name for name in PLAN_FILENAMES if os.path.isfile(os.path.join(directory, name))] + + if not found: + raise DiscoveryError( + f"No plan document found in {os.path.abspath(directory)}. Expected one of " + f"{' or '.join(PLAN_FILENAMES)}. Either write one with " + f"`terraform show -json tfplan > plan.json`, point --plan-file at the binary plan, or " + f"pass --input-path explicitly." + ) + + if len(found) > 1: + raise DiscoveryError( + f"Found {' and '.join(found)} in {os.path.abspath(directory)} and cannot tell which to " + f"evaluate. Pass --input-path to choose." + ) + + return os.path.join(directory, found[0]) + + +def _resolve_binary(explicit=None): + """ + Find a terraform/tofu binary, preferring the real one over a wrapper. + + `hashicorp/setup-terraform` installs a JS wrapper as `terraform` and moves the real binary to + `terraform-bin`. That wrapper calls `core.setOutput('stdout', ...)`, so invoking it for + `show -json` appends the *entire plan* to $GITHUB_OUTPUT -- an unmasked plan written to a file + every later step in the job can read. `opentofu/setup-opentofu` does the same with `tofu-bin`. + + So the `-bin` names come first, and the wrappers are only a last resort. + """ + if explicit: + return explicit + + candidates = [] + for env_var, binary in (("TERRAFORM_CLI_PATH", "terraform-bin"), ("TOFU_CLI_PATH", "tofu-bin")): + directory = os.environ.get(env_var) + if directory: + candidates.append(os.path.join(directory, binary)) + candidates += ["terraform-bin", "tofu-bin", "terraform", "tofu"] + + for candidate in candidates: + if os.path.isabs(candidate): + if os.path.isfile(candidate) and os.access(candidate, os.X_OK): + return candidate + else: + resolved = shutil.which(candidate) + if resolved: + return resolved + + raise DiscoveryError( + "No terraform or tofu binary found on PATH. Pass --terraform-bin, or write the plan JSON " + "yourself and pass --input-path." + ) + + +def terraform_show_json(plan_file, workdir=None, binary=None): + """ + Render a binary plan to JSON in memory. + + The point is that nothing unmasked touches the disk: the JSON is parsed straight off the pipe + and handed to the masker. stdout is never logged, for the same reason. + """ + executable = _resolve_binary(binary) + if not binary: + # setup-terraform and setup-opentofu both install a wrapper that echoes stdout into + # $GITHUB_OUTPUT, and both advertise it the same way. Guarding only the terraform spelling + # left the opentofu one to copy the whole unmasked plan into the step output. + for env_var, wrapper in (("TERRAFORM_CLI_PATH", "terraform"), ("TOFU_CLI_PATH", "tofu")): + if os.environ.get(env_var) and os.path.basename(executable) == wrapper: + # Only reachable if the -bin names were all absent, which means the wrapper was + # installed without its usual layout. Say so rather than silently leaking the plan. + raise DiscoveryError( + f"{env_var} is set but no {wrapper}-bin was found beside it, so the only " + f"{wrapper} on PATH is the setup wrapper. Running it would copy the whole plan " + f"into $GITHUB_OUTPUT. Pass --terraform-bin with the real binary." + ) + + directory = workdir or os.path.dirname(os.path.abspath(plan_file)) or "." + plan_arg = os.path.abspath(plan_file) + + try: + completed = subprocess.run( + [executable, "show", "-json", plan_arg], + cwd=directory, + capture_output=True, + timeout=300, + ) + except (OSError, subprocess.TimeoutExpired) as e: + raise DiscoveryError(f"Could not run `{executable} show -json`: {e}") + + if completed.returncode != 0: + stderr = completed.stderr.decode("utf-8", "replace").strip()[:2000] + raise DiscoveryError(f"`{executable} show -json` failed (exit {completed.returncode}): {stderr}") + + try: + return json.loads(completed.stdout) + except json.JSONDecodeError as e: + # Deliberately does not echo stdout: on the wrapper path it would be the whole plan. + raise DiscoveryError(f"`{executable} show -json` did not produce JSON: {e}") diff --git a/src/tirith/platform/redact.py b/src/tirith/platform/redact.py new file mode 100644 index 00000000..321eb2b6 --- /dev/null +++ b/src/tirith/platform/redact.py @@ -0,0 +1,679 @@ +""" +Slim and mask terraform documents before they leave the runner. + +This runs client-side on purpose. Once bytes reach StackGuardian the exposure has already +happened, so masking on the server would be theatre. Everything here is a pure function over +parsed JSON so it can be tested exhaustively. + +A caveat worth stating plainly, and repeated in the README: terraform's `*_sensitive` markers are +NOT exhaustive. A value that flows through `locals`, or comes from a provider that did not mark +its schema, arrives marked `false` and will not be masked by marker-driven redaction. Slimming and +the `variables` drop below exist partly to limit that blast radius. +""" + +import copy + +SENTINEL = "__SG_REDACTED__" + +# Top-level plan sections tirith's terraform_plan provider never reads, verified against +# providers/terraform_plan/handler.py: +# +# resource_changes -> attribute / action / count operations +# configuration -> direct_dependencies, direct_references, provider_config (KEPT) +# terraform_version -> terraform_version operation +# +# `planned_values` is the dangerous one. It mirrors every resource's values in a second place and +# carries NO sensitivity markers of its own, so marker-driven redaction of `resource_changes` +# leaves the same secret in plaintext here. Dropping it is lossless for evaluation and closes that +# hole; a real plan leaked a `local_sensitive_file` body through exactly this path. +SLIM_DROP_KEYS = ("prior_state", "planned_values") + +# Provider blocks whose `expressions` can hold hardcoded credentials. `configuration` cannot be +# dropped wholesale -- three tirith operations read it -- so the credential-bearing part is +# scrubbed instead, keeping the two fields provider_config_operator actually consults. +_PROVIDER_CONFIG_KEEP = ("name", "full_name", "version_constraint", "module_address", "alias") + + +def slim_plan(plan): + """ + Drop plan sections that are irrelevant to evaluation. + + Typically removes 60-90% of the bytes. `configuration` is deliberately retained but scrubbed + (see `_scrub_configuration`), because dropping it would silently break the + `direct_dependencies`, `direct_references` and `provider_config` operations -- policies would + stop finding what they are looking for rather than failing loudly. + """ + if not isinstance(plan, dict): + return plan + + slimmed = {k: v for k, v in plan.items() if k not in SLIM_DROP_KEYS} + if isinstance(slimmed.get("configuration"), dict): + slimmed["configuration"] = _scrub_configuration(slimmed["configuration"]) + return slimmed + + +def _scrub_configuration(configuration): + """ + Strip credential-bearing expressions from `configuration` while keeping what tirith reads. + + Two places hold literals, and both have to be scrubbed: + + `provider_config[].expressions` -- `provider_config_operator` reads only `version_constraint` + and `expressions.region.constant_value`, so access keys, tokens and assume-role blocks can go. + + `root_module.resources[].expressions[].constant_value` -- every literal written in the HCL, + including a hardcoded password. This is a third instance of the `planned_values` pattern: a + place values live that carries no sensitivity markers, so marker-driven masking of + `resource_changes` never touches it. Caught in QA -- a `local_sensitive_file` body was masked + in `resource_changes` and sat in plaintext here in the same document. + + Dropping `constant_value` is lossless: `direct_references_operator` reads only `references` + from these expressions, and `direct_dependencies_operator` reads only `depends_on` + (providers/terraform_plan/handler.py:329, :385-388). + """ + scrubbed = dict(configuration) + + provider_config = scrubbed.get("provider_config") + if isinstance(provider_config, dict): + cleaned = {} + for name, block in provider_config.items(): + if not isinstance(block, dict): + cleaned[name] = block + continue + kept = {k: v for k, v in block.items() if k in _PROVIDER_CONFIG_KEEP} + region = (block.get("expressions") or {}).get("region") + if region is not None: + kept["expressions"] = {"region": region} + cleaned[name] = kept + scrubbed["provider_config"] = cleaned + + root_module = scrubbed.get("root_module") + if isinstance(root_module, dict): + scrubbed["root_module"] = _scrub_config_module(root_module) + + return scrubbed + + +def _scrub_config_module(module): + """Recursively drop literal values from a configuration module, keeping the reference graph.""" + scrubbed = dict(module) + + resources = scrubbed.get("resources") + if isinstance(resources, list): + scrubbed["resources"] = [_scrub_config_resource(r) for r in resources] + + # Child modules nest the same shape under module_calls[].module. + module_calls = scrubbed.get("module_calls") + if isinstance(module_calls, dict): + calls = {} + for name, call in module_calls.items(): + if isinstance(call, dict): + if isinstance(call.get("module"), dict): + call = {**call, "module": _scrub_config_module(call["module"])} + else: + call = dict(call) + # A module's own arguments are literals too. Dropped whether or not the call + # carries an inlined `module` body -- it did not when the module came from a + # registry or a git source, which is the common case, and the arguments passed to + # it are literals either way. + call.pop("expressions", None) + calls[name] = call + scrubbed["module_calls"] = calls + + # Variable defaults and output values are literals with no operation reading them. + for section in ("variables", "outputs"): + if isinstance(scrubbed.get(section), dict): + scrubbed[section] = _scrub_config_section(scrubbed[section]) + + return scrubbed + + +def _scrub_config_resource(resource): + if not isinstance(resource, dict): + return resource + + scrubbed = dict(resource) + + expressions = scrubbed.get("expressions") + if isinstance(expressions, dict): + scrubbed["expressions"] = {k: _keep_references(v) for k, v in expressions.items()} + + # A provisioner carries its own expressions one level down -- `connection.password`, and the + # `inline` script itself. Scrubbing only the resource's own expressions left those verbatim, + # and a provisioner block is exactly where a password tends to be written literally. + provisioners = scrubbed.get("provisioners") + if isinstance(provisioners, list): + scrubbed["provisioners"] = [_scrub_config_resource(p) for p in provisioners] + + # count/for_each are expressions in their own right, and a `for_each` over a map of literals + # carries those literals. + for key in ("count_expression", "for_each_expression"): + if key in scrubbed: + scrubbed[key] = _keep_references(scrubbed[key]) + + return scrubbed + + +def _keep_references(expression): + """ + Reduce one expression to just its `references`, dropping every literal. + + Terraform nests expressions arbitrarily: a block argument is a dict of expressions, and a + repeated block is a list of them, so this recurses rather than looking one level deep. + """ + if isinstance(expression, list): + return [_keep_references(item) for item in expression] + if not isinstance(expression, dict): + return expression + if "references" in expression or "constant_value" in expression: + # A leaf: keep only the reference graph. + return {"references": expression["references"]} if "references" in expression else {} + return {k: _keep_references(v) for k, v in expression.items()} + + +def _scrub_config_section(section): + """Drop `default` / `expression` literals from variables and outputs.""" + cleaned = {} + for name, entry in section.items(): + if isinstance(entry, dict): + entry = {k: v for k, v in entry.items() if k not in ("default", "expression", "value")} + cleaned[name] = entry + return cleaned + + +def _mask_by_marker(value, marker): + """ + Walk `value` alongside terraform's parallel sensitivity structure `marker`. + + A marker node of `true` masks the whole subtree beneath it. Dicts and lists are walked in + lockstep; anything else is returned untouched. + """ + if marker is True: + return SENTINEL + + if isinstance(marker, dict) and isinstance(value, dict): + return {k: _mask_by_marker(v, marker.get(k)) for k, v in value.items()} + + if isinstance(marker, list) and isinstance(value, list): + # Terraform emits a marker list positionally aligned with the value list. A shorter + # marker list means the tail is not sensitive. + return [_mask_by_marker(item, marker[i] if i < len(marker) else None) for i, item in enumerate(value)] + + return value + + +# A value shorter than this is not swept. `_sweep_known_secrets` replaces exact string matches +# everywhere, and a two-character secret would also match ids, regions and resource names -- mangling +# the document the policies then evaluate. A real credential is longer than this; a two-character one +# that leaks is the lesser harm against breaking every policy on the plan. +MIN_SWEPT_SECRET_LENGTH = 6 + + +def _collect_sensitive_values(value, marker, found): + """Gather the plaintext strings terraform marked sensitive, so they can be swept elsewhere.""" + if marker is True: + if isinstance(value, str) and len(value) >= MIN_SWEPT_SECRET_LENGTH: + found.add(value) + elif isinstance(value, (dict, list)): + _collect_all_strings(value, found) + return + + if isinstance(marker, dict) and isinstance(value, dict): + for key, item in value.items(): + _collect_sensitive_values(item, marker.get(key), found) + elif isinstance(marker, list) and isinstance(value, list): + for index, item in enumerate(value): + _collect_sensitive_values(item, marker[index] if index < len(marker) else None, found) + + +def _collect_all_strings(node, found): + """Every string under a subtree terraform marked sensitive wholesale.""" + if isinstance(node, dict): + for item in node.values(): + _collect_all_strings(item, found) + elif isinstance(node, list): + for item in node: + _collect_all_strings(item, found) + elif isinstance(node, str) and len(node) >= MIN_SWEPT_SECRET_LENGTH: + found.add(node) + + +def _sweep_known_secrets(node, secrets): + """ + Replace any value terraform told us was sensitive *somewhere* with the sentinel *everywhere*. + + The markers alone are not enough. A provider that computes a mirror of an attribute does not + inherit its sensitivity: an `aws_instance` with a sensitive value in `tags` is marked + `after_sensitive.tags.Password = true`, while `after_sensitive.tags_all` comes back `{}` even + though `tags_all` holds the identical plaintext. Every AWS resource with tags has `tags_all`, so + that single gap leaks any secret ever used in a tag. + + Caught by an end-to-end test that downloaded the uploaded bundle and grepped it, not by the unit + suite -- which asserted the markers were honoured, and they were. + + Exact string matches only: it cannot know that a *substring* is the secret without guessing, and a + guess here corrupts the document the policies read. + """ + if not secrets: + return node + if isinstance(node, dict): + return {k: _sweep_known_secrets(v, secrets) for k, v in node.items()} + if isinstance(node, list): + return [_sweep_known_secrets(item, secrets) for item in node] + if isinstance(node, str) and node in secrets: + return SENTINEL + return node + + +def _mask_resource_change(resource_change): + """Mask one `resource_changes`/`resource_drift` entry by its own before/after markers.""" + if not isinstance(resource_change, dict): + return resource_change + + masked = dict(resource_change) + change = masked.get("change") + if isinstance(change, dict): + masked_change = dict(change) + for value_key, marker_key in (("before", "before_sensitive"), ("after", "after_sensitive")): + if value_key in masked_change: + masked_change[value_key] = _mask_by_marker(masked_change[value_key], masked_change.get(marker_key)) + masked["change"] = masked_change + return masked + + +def redact_plan(plan): + """ + Slim, then mask every value terraform flagged sensitive, then drop root `variables`. + + `variables` goes wholesale because the plan does not reliably mark which root variables were + declared `sensitive = true` -- so the only safe assumption is that all of them might be. + """ + plan = slim_plan(plan) + if not isinstance(plan, dict): + return plan + + redacted = dict(plan) + + # Collect the sensitive plaintext BEFORE masking replaces it, and before `variables` is dropped -- + # a sensitive root variable is often the origin of the value that reappears elsewhere unmarked. + secrets = set() + for section in ("resource_changes", "resource_drift"): + for entry in redacted.get(section) or []: + change = (entry or {}).get("change") if isinstance(entry, dict) else None + if isinstance(change, dict): + for value_key, marker_key in (("before", "before_sensitive"), ("after", "after_sensitive")): + _collect_sensitive_values(change.get(value_key), change.get(marker_key), secrets) + for name, variable in (redacted.get("variables") or {}).items(): + if isinstance(variable, dict): + _collect_all_strings(variable.get("value"), secrets) + + redacted.pop("variables", None) + + # resource_drift has the same shape and the same sensitivity markers as resource_changes, and + # terraform emits it whenever a refresh finds drift -- so a masked resource_changes sitting + # beside an unmasked resource_drift shipped the same secret in plaintext one key away. + for section in ("resource_changes", "resource_drift"): + entries = redacted.get(section) + if isinstance(entries, list): + redacted[section] = [_mask_resource_change(entry) for entry in entries] + + output_changes = redacted.get("output_changes") + if isinstance(output_changes, dict): + redacted["output_changes"] = {name: _redact_output_change(change) for name, change in output_changes.items()} + + # Rebuild planned_values from what we just masked. slim_plan dropped terraform's own copy + # because it carries no sensitivity markers; this one is derived from the masked + # resource_changes, so it holds the same redacted values. + planned_values = rebuild_planned_values(redacted.get("resource_changes")) + if planned_values: + redacted["planned_values"] = planned_values + + # Last, over the whole document: anything terraform called sensitive somewhere is masked + # everywhere, including the unmarked provider-computed mirrors the markers miss. + return _sweep_known_secrets(redacted, secrets) + + +def rebuild_planned_values(masked_resource_changes): + """ + Reconstruct `planned_values` from already-masked `resource_changes`. + + Infracost and Checkov both read `planned_values` and nothing else -- give them a plan without + it and they return a clean, empty, entirely wrong answer. Measured against infracost 0.10.27 + with a real API key: the same t3.medium prices at $39.80 with the key present and $0.00 + without, differing only by this one section. + + Terraform's own copy cannot be shipped: it mirrors every value with NO sensitivity markers, so + masking `resource_changes` leaves the same secret in plaintext there -- a real plan leaked a + `local_sensitive_file` body through exactly that path. This rebuild sidesteps that because it + reads the *masked* values, after `_mask_by_marker` has run over them. + + Only `after` is used, and only for resources that will exist. A destroy has no planned value, + and `before` is the pre-change state that `prior_state` carries -- which is dropped for the + same marker-less reason. + """ + if not isinstance(masked_resource_changes, list): + return None + + root = {"resources": [], "child_modules": []} + modules = {} + + for resource_change in masked_resource_changes: + if not isinstance(resource_change, dict): + continue + change = resource_change.get("change") + if not isinstance(change, dict): + continue + if "delete" in (change.get("actions") or []) and "create" not in (change.get("actions") or []): + # Nothing is planned to exist, so there is nothing to price or scan. + continue + after = change.get("after") + if after is None: + continue + + resource = { + key: resource_change[key] + for key in ("address", "mode", "type", "name", "index", "provider_name") + if key in resource_change + } + resource["values"] = after + + module_address = resource_change.get("module_address") + if module_address: + modules.setdefault(module_address, {"address": module_address, "resources": []})["resources"].append( + resource + ) + else: + root["resources"].append(resource) + + if modules: + # Flat rather than a true nesting tree. Verified equivalent for pricing, and both tools + # address resources by their full `address`, which already encodes the module path. + root["child_modules"] = sorted(modules.values(), key=lambda m: m["address"]) + else: + root.pop("child_modules") + + if not root["resources"] and not root.get("child_modules"): + return None + + return {"root_module": root} + + +def _redact_output_change(change): + """ + Mask a sensitive output's before/after values. + + Terraform spells the marker differently across versions: older plans carry a single + `sensitive`, newer ones carry `before_sensitive` / `after_sensitive` per side. Checking only + `sensitive` silently missed every modern plan, so all three are honoured -- and each side is + masked independently, since an output can become sensitive without having been so before. + + Only keys that are actually present are replaced. Adding an `after` to a create whose value is + still unknown (`after_unknown: true`) would invent data the plan never contained. + """ + if not isinstance(change, dict): + return change + + masked = dict(change) + whole = bool(change.get("sensitive")) + + for side in ("before", "after"): + if side not in masked: + continue + if whole or change.get(f"{side}_sensitive") is True: + masked[side] = SENTINEL + + return masked + + +def redact_state(state): + """ + Mask a terraform state document. + + State is more dangerous than a plan: it holds every resource attribute in plaintext, including + values no plan would surface. Two rules, matching what the platform's terraform step applies: + + - `outputs[k].sensitive` is true -> replace that output's value + - each key named in an instance's `sensitive_attributes` -> replace that attribute + + Handles BOTH shapes a caller can plausibly hand us: + + - the raw state (`terraform state pull`): top-level `resources` / `outputs`, with each + instance naming its own `sensitive_attributes`; + - `terraform show -json `: resources nested under `values.root_module.resources`, with + sensitivity carried in a parallel `sensitive_values` tree. + + Handling only the first was a silent leak. The function returned the document unchanged for the + second -- no error, no warning -- so a state produced with `show -json`, which is the natural + way to get a readable one, shipped every attribute in plaintext. Caught by an end-to-end run, + not by a unit test, because the unit tests all used the shape the code already understood. + """ + if not isinstance(state, dict): + return state + + redacted = dict(state) + + # Every plaintext we are about to mask, collected as we go and swept from the whole document at the + # end -- the same two-pass shape as redact_plan, and for the same reason. + # + # Marker-driven masking alone is not enough, because a provider writes computed *mirrors* of an + # attribute with no sensitivity marker of their own. The confirmed case is `tags_all`: a secret in + # `tags.Password` is masked there and shipped in plaintext one key away. A pen test found this exact + # hole in state after the equivalent had been closed for plans -- and state is the worse place for + # it, since state carries every attribute of every resource and the bundle is retained. + secrets = set() + + values = redacted.get("values") + if isinstance(values, dict): + redacted["values"] = _redact_show_json_values(values, secrets) + + outputs = redacted.get("outputs") + if isinstance(outputs, dict): + masked_outputs = {} + for name, output in outputs.items(): + if isinstance(output, dict) and output.get("sensitive"): + _collect_all_strings(output.get("value"), secrets) + masked_outputs[name] = {**output, "value": SENTINEL} + else: + masked_outputs[name] = output + redacted["outputs"] = masked_outputs + + resources = redacted.get("resources") + if isinstance(resources, list): + redacted["resources"] = [_redact_state_resource(r, secrets) for r in resources] + + return _sweep_known_secrets(redacted, secrets) + + +def _redact_show_json_values(values, secrets=None): + """ + Mask the `values` tree of `terraform show -json ` output. + + Same marker convention as a plan: a parallel `sensitive_values` tree whose truthy leaves name + the attributes to replace, so _mask_by_marker does the work. Recurses through child_modules, + since a module's resources are nested rather than flattened. + """ + if not isinstance(values, dict): + return values + + if secrets is None: + secrets = set() + + masked = dict(values) + root = masked.get("root_module") + if isinstance(root, dict): + masked["root_module"] = _redact_show_json_module(root, secrets) + + outputs = masked.get("outputs") + if isinstance(outputs, dict): + masked_outputs = {} + for name, o in outputs.items(): + if isinstance(o, dict) and o.get("sensitive"): + _collect_all_strings(o.get("value"), secrets) + masked_outputs[name] = {**o, "value": SENTINEL} + else: + masked_outputs[name] = o + masked["outputs"] = masked_outputs + return masked + + +def _redact_show_json_module(module, secrets=None): + if not isinstance(module, dict): + return module + + if secrets is None: + secrets = set() + + masked = dict(module) + + resources = masked.get("resources") + if isinstance(resources, list): + out = [] + for resource in resources: + if not isinstance(resource, dict): + out.append(resource) + continue + entry = dict(resource) + if "values" in entry: + # Collect before masking. The marker convention here is identical to a plan's, so this + # is the same call redact_plan makes over `change.before` / `change.after`. + _collect_sensitive_values(entry["values"], entry.get("sensitive_values"), secrets) + entry["values"] = _mask_by_marker(entry["values"], entry.get("sensitive_values")) + out.append(entry) + masked["resources"] = out + + children = masked.get("child_modules") + if isinstance(children, list): + masked["child_modules"] = [_redact_show_json_module(c, secrets) for c in children] + + return masked + + +def _redact_state_resource(resource, secrets=None): + if not isinstance(resource, dict): + return resource + + if secrets is None: + secrets = set() + + instances = resource.get("instances") + if not isinstance(instances, list): + return resource + + masked_instances = [] + for instance in instances: + if not isinstance(instance, dict): + masked_instances.append(instance) + continue + + masked = dict(instance) + attributes = masked.get("attributes") + sensitive_attributes = masked.get("sensitive_attributes") or [] + + if isinstance(attributes, dict) and sensitive_attributes: + masked_attributes = copy.deepcopy(attributes) + for sensitive_attribute in sensitive_attributes: + steps = _attribute_steps(sensitive_attribute) + # Read from the untouched original, not from the copy being masked: once the first path + # is masked the copy holds the sentinel there, and sweeping for that would do nothing. + plaintext = _read_attribute_path(attributes, steps) + if plaintext is not _ABSENT: + _collect_all_strings(plaintext, secrets) + _mask_attribute_path(masked_attributes, steps) + masked["attributes"] = masked_attributes + + masked_instances.append(masked) + + return {**resource, "instances": masked_instances} + + +def _attribute_steps(sensitive_attribute): + """ + Normalise one `sensitive_attributes` entry into a list of path steps. + + Terraform writes each entry as a PATH -- a list of steps -- not a single key: + + [[{"type": "get_attr", "value": "content_base64"}], + [{"type": "get_attr", "value": "content"}]] + + Reading only the flat forms silently masked nothing at all on real state, because a list is + neither a dict nor a string. Verified against `terraform state pull` output for a + `local_sensitive_file`; the earlier unit tests passed only because their fixture invented the + flat shape. + + The two flat forms are still accepted: some providers and older state versions emit them. + """ + if isinstance(sensitive_attribute, list): + entries = sensitive_attribute + else: + entries = [sensitive_attribute] + + steps = [] + for entry in entries: + if isinstance(entry, dict): + steps.append(entry.get("value")) + elif isinstance(entry, (str, int)): + steps.append(entry) + else: + # An unrecognised step means the path cannot be trusted; masking a guessed location + # would be worse than reporting nothing. + return [] + return steps + + +_ABSENT = object() + + +def _read_attribute_path(container, steps): + """ + Return the value at `steps` within `container`, or `_ABSENT`. + + The mirror of `_mask_attribute_path` below, and deliberately the same walk: the two have to agree + on what a path means, or the sweep collects a different value from the one that was masked. + """ + if not steps: + return _ABSENT + + node = container + for step in steps: + if isinstance(node, dict) and step in node: + node = node[step] + elif isinstance(node, list) and isinstance(step, int) and 0 <= step < len(node): + node = node[step] + else: + return _ABSENT + return node + + +def _mask_attribute_path(container, steps): + """ + Replace the value at `steps` within `container` with the sentinel. + + A path may descend through nested objects and list indices -- `[{"get_attr": "config"}, + {"index": 0}, {"get_attr": "token"}]` -- so this walks rather than assuming one level. + """ + if not steps: + return + + *parents, leaf = steps + node = container + for step in parents: + if isinstance(node, dict) and step in node: + node = node[step] + elif isinstance(node, list) and isinstance(step, int) and 0 <= step < len(node): + node = node[step] + else: + return + + if isinstance(node, dict) and leaf in node: + node[leaf] = SENTINEL + elif isinstance(node, list) and isinstance(leaf, int) and 0 <= leaf < len(node): + node[leaf] = SENTINEL + + +def count_redactions(document): + """Count sentinel occurrences, for the attestation the action sends with the upload.""" + if isinstance(document, dict): + return sum(count_redactions(v) for v in document.values()) + if isinstance(document, list): + return sum(count_redactions(v) for v in document) + return 1 if document == SENTINEL else 0 diff --git a/src/tirith/platform/regions.py b/src/tirith/platform/regions.py new file mode 100644 index 00000000..06df0a8b --- /dev/null +++ b/src/tirith/platform/regions.py @@ -0,0 +1,145 @@ +""" +StackGuardian regions, and the one place URLs are resolved. + +A region is a well-known (API, dashboard) pair, so asking a caller for both URLs is asking them to +keep two constants in sync for no reason. Getting it half right is the common failure: overriding +only the API leaves every run link in every PR comment pointing at the wrong environment, which +looks like a broken integration rather than a misconfiguration. + +`region` is the same identifier the Raycast extension uses, so a user who has configured one +recognises the other. + +Note the API base here excludes `/api/v1`, matching Raycast, sg-cli and the terraform provider. +`--api-url` and `$SG_BASE_URL` have always included it, and `normalize_api_url` accepts both -- a +value exported for sg-cli previously produced 404s from tirith. +""" + +import collections + +Region = collections.namedtuple("Region", "id name api_base app_base") + +# Only production regions are listed. Internal environments are reachable through --api-url / +# $SG_BASE_URL, which is also what a self-hosted or vanity host (api..stackguardian.io) +# needs, so they are supported rather than merely tolerated. +# +# The dashboard uses a third spelling for the same regions ('eu1-europe' / 'us1-east'). These ids are +# the CLI and action spelling; there are two regions, not four. +REGIONS = ( + Region("eu", "Europe", "https://api.app.stackguardian.io", "https://app.stackguardian.io"), + Region("us", "United States", "https://api.us.stackguardian.io", "https://us.stackguardian.io"), +) + +DEFAULT_REGION_ID = "eu" + +REGION_IDS = tuple(region.id for region in REGIONS) + +API_PATH = "/api/v1" + + +def by_id(region_id): + """ + Look up a region, raising on an unknown id. + + Deliberately not the "fall back to the first region" behaviour the Raycast extension uses: + here a typo would silently evaluate a US org's infrastructure against production EU, and the + only symptom would be an authentication error the user cannot explain. + """ + for region in REGIONS: + if region.id == region_id: + return region + raise ValueError(f"Unknown region '{region_id}'. Valid regions: {', '.join(REGION_IDS)}") + + +def normalize_api_url(api_url): + """ + Accept an API base with or without the `/api/v1` suffix. + + tirith's own flag has always included it; every other StackGuardian client omits it. Rejecting + one spelling would be a papercut for anyone who has already exported SG_BASE_URL for sg-cli. + """ + trimmed = (api_url or "").rstrip("/") + if not trimmed: + return trimmed + if trimmed.endswith(API_PATH): + return trimmed + return f"{trimmed}{API_PATH}" + + +def by_api_url(api_url): + """Find the region an API URL belongs to, tolerating the `/api/v1` suffix. None if unknown.""" + normalized = normalize_api_url(api_url) + for region in REGIONS: + if normalized == normalize_api_url(region.api_base): + return region + return None + + +def resolve(region_id=None, api_url=None, dashboard_url=None, env=None): + """ + Resolve (api_url, dashboard_url, warnings) from a region, explicit URLs and the environment. + + Precedence, highest first: + + 1. explicit --api-url / --dashboard-url + 2. --region + 3. $SG_BASE_URL / $SG_DASHBOARD_URL, then $SG_REGION + 4. the default region + + Explicit URLs beat a region because they are the only way to reach a self-hosted install, so + they have to keep working permanently rather than as a deprecation shim. Passing both a region + and an explicit URL is a caller error -- they contradict each other, and silently picking one + would hide it. + + A URL environment variable beats $SG_REGION rather than erroring: environment is inherited + config the caller may not control, and failing a CI run over it would be unhelpful. + """ + env = {} if env is None else env + warnings = [] + + env_api_url = env.get("SG_BASE_URL") + env_dashboard_url = env.get("SG_DASHBOARD_URL") + env_region_id = env.get("SG_REGION") + + if region_id and (api_url or dashboard_url): + which = " and ".join( + name for name, value in (("--api-url", api_url), ("--dashboard-url", dashboard_url)) if value + ) + raise ValueError(f"--region and {which} cannot be combined; they set the same thing") + + effective_region_id = region_id or env_region_id + if effective_region_id and not region_id and (env_api_url or env_dashboard_url): + warnings.append( + f"both $SG_REGION and $SG_BASE_URL/$SG_DASHBOARD_URL are set; using the URLs and " + f"ignoring region '{effective_region_id}'" + ) + effective_region_id = None + + if effective_region_id: + region = by_id(effective_region_id) + return normalize_api_url(region.api_base), region.app_base, warnings + + resolved_api = api_url or env_api_url + resolved_dashboard = dashboard_url or env_dashboard_url + default_region = by_id(DEFAULT_REGION_ID) + + if not resolved_api and not resolved_dashboard: + return normalize_api_url(default_region.api_base), default_region.app_base, warnings + + if not resolved_api: + resolved_api = default_region.api_base + + if not resolved_dashboard: + # The footgun this function exists for: setting only the API leaves every run link pointing + # at the default environment. Infer the dashboard when the API is a region we know, and say + # so out loud when it is not. + matched = by_api_url(resolved_api) + if matched: + resolved_dashboard = matched.app_base + else: + resolved_dashboard = default_region.app_base + warnings.append( + f"no dashboard URL given and '{resolved_api}' is not a known region, so run links " + f"will point at {resolved_dashboard}; pass --dashboard-url to fix them" + ) + + return normalize_api_url(resolved_api), resolved_dashboard.rstrip("/"), warnings diff --git a/src/tirith/platform/report.py b/src/tirith/platform/report.py new file mode 100644 index 00000000..ffb15cd5 --- /dev/null +++ b/src/tirith/platform/report.py @@ -0,0 +1,442 @@ +""" +Turn PolicyEvalResults into a PR comment body, a check-run summary, and a verdict. + +Pure functions over the results document so the layout and the truncation arithmetic can be tested +without touching a network. +""" + +import html + +FAIL = "FAIL" +WARN = "WARN" +PASS = "PASS" +APPROVAL_REQUIRED = "APPROVAL_REQUIRED" + +# Anything the step reports that is not one of the four above. It is counted separately and treated +# as unresolved rather than folded into any of them: a result this module does not understand is not +# evidence of a pass, and bucketing it under a key `verdict` never inspects made it one. +UNKNOWN = "UNKNOWN" + +# GitHub rejects an issue-comment body over 65536 characters and a check-run output.summary over +# 65535. Budget well under both: the count that matters is characters after rendering, and a +# 422 at the end of a run is a bad way to find out. +COMMENT_LIMIT = 60000 + +_ICONS = {FAIL: "❌", WARN: "⚠️", APPROVAL_REQUIRED: "⏳", PASS: "✅", UNKNOWN: "❓"} + + +def summarize(policy_results): + """ + Collapse the results into counts plus a flat finding list. + + A rule marked `skip` carries no verdict, so it is counted separately rather than being + folded into passes -- reporting a skipped control as passing is the kind of quiet + inaccuracy this whole design exists to avoid. + """ + counts = {FAIL: 0, WARN: 0, APPROVAL_REQUIRED: 0, PASS: 0, "SKIPPED": 0, UNKNOWN: 0} + findings = [] + + for policy_id, rules in sorted((policy_results or {}).items()): + for rule in rules or []: + if rule.get("skip"): + counts["SKIPPED"] += 1 + findings.append( + { + "policy_id": policy_id, + "rule_name": rule.get("rule_name", ""), + "result": "SKIPPED", + "messages": [], + "resources": [], + } + ) + continue + + # No default of PASS: a rule the step wrote without a `result`, or with one this module + # does not know, is unresolved. Defaulting to PASS turned "we cannot tell" into a clean + # bill of health, and an unrecognised value landed in a count key `verdict` never reads, + # so it disappeared entirely. + result = rule.get("result") + if result not in (FAIL, WARN, APPROVAL_REQUIRED, PASS): + result = UNKNOWN + counts[result] = counts.get(result, 0) + 1 + messages, resources = _extract_detail(rule) + findings.append( + { + "policy_id": policy_id, + "rule_name": rule.get("rule_name", ""), + "result": result, + "messages": messages, + "resources": resources, + } + ) + + return counts, findings + + +def _extract_detail(rule): + """Pull human-readable messages and resource addresses out of a rule's evaluations.""" + messages = [] + resources = [] + + for entry in (rule.get("evaluations") or {}).get("fails") or []: + if "exec_err" in entry: + # An engine/config problem rather than a policy violation -- surfaced verbatim so a + # malformed policy is not mistaken for a real finding. + messages.append(f"engine: {entry['exec_err']}") + continue + + # Checkov findings are shaped differently from tirith's: {"description", "keys"} rather + # than a list under "result". Reading only the tirith shape rendered a Checkov policy as an + # empty
block -- a dozen real findings, silently blank, in the one place a + # reviewer looks. + # + # Both shapes are read here rather than dispatched between, because an entry can carry both + # keys. A tirith rule sets `description` to "" when the policy declares none and puts the + # finding under `result`; branching on the *presence* of `description` therefore matched the + # Checkov shape, found nothing to say, and skipped the `result` loop -- reproducing exactly + # the blank block above for cost rules. This is additive: a Checkov entry has no `result`, + # so its loop is a no-op. + description = entry.get("description") + if description: + messages.append(description) + for key in entry.get("keys") or []: + # `aws_instance.app.root_block_device` -> `aws_instance.app`. The suffix is the + # attribute the check looked at; the address is what a reviewer navigates by. + address = _resource_address(key) + if address and address not in resources: + resources.append(address) + + for evaluation in entry.get("result") or []: + message = evaluation.get("message") + if message: + messages.append(message) + # Only the terraform_plan provider populates meta; others set it to None. + meta = evaluation.get("meta") or {} + address = meta.get("address") if isinstance(meta, dict) else None + if address and address not in resources: + resources.append(address) + + return messages, resources + + +def _resource_address(key): + """ + Reduce a Checkov evaluated key to the resource address it belongs to. + + Checkov reports `..`, and the attribute path can be arbitrarily + deep (`aws_s3_bucket.data.rule.apply_server_side_encryption_by_default.sse_algorithm`). The + first two segments are the address; everything after is what the check inspected. + """ + if not isinstance(key, str): + return None + parts = key.split(".") + if len(parts) < 2: + return None + return ".".join(parts[:2]) + + +def verdict(counts, run_status): + """ + Reduce counts and run status to one word. + + failed | warned | passed | no-policies | errored + + `errored` covers a run that never produced a verdict -- an ERRORED/CANCELLED run, or results + that came back empty. It is deliberately distinct from `failed` so the caller can tell "a + policy said no" from "we do not know", and never conflate either with a pass. + + A policy carrying `onFail: APPROVAL_REQUIRED` warns; it does not gate. That is a deliberate + interim position, because for these runs there is nothing to approve. The step exits 0 (it never + uses exit 11), so the run reaches COMPLETED, and the run controller engages an approval only on + exit 11 and skips it on the last step anyway -- and a policy-only run has exactly one step. So + the approval intent arrives as a count on an already-finished run, with no approval to act on. + Blocking on it produced a red check with nothing to click. + + The count, the icon and the "N need approval" phrase in the headline all survive, so the policy + author's intent is still visible in the comment. Gating on it properly needs a run that stays + open, an approve action on it, and this client re-polling afterwards -- none of which exist yet. + + Run status APPROVAL_REQUIRED means the platform itself paused the run. It is ranked by the same + ladder and then floored, rather than short-circuited: an early return there let a paused run + carrying a FAIL report `warned`, which is the one direction that must never happen. And because + a paused run did not finish, it can never rank better than `warned` either -- the policies that + would have run after the pause did not, so "everything passed" is not something we know. + + A rule whose result this module does not recognise counts as UNKNOWN and lands in `errored`. + "We cannot tell" is not a pass. + """ + if run_status not in ("COMPLETED", "APPROVAL_REQUIRED"): + return "errored" + + paused = run_status == "APPROVAL_REQUIRED" + + if counts.get(FAIL): + return "failed" + # An unreadable result outranks a warning: part of the evaluation is unaccounted for. + if counts.get(UNKNOWN): + return "errored" + if counts.get(APPROVAL_REQUIRED) or counts.get(WARN): + return "warned" + if counts.get(PASS) or counts.get("SKIPPED"): + return "warned" if paused else "passed" + # No policy results at all. On a COMPLETED run that means nothing was in scope -- worth saying, + # rather than implying a clean bill of health. On a paused run it means the evaluation never got + # far enough to produce any, which is not "nothing in scope" but "we do not know". + return "errored" if paused else "no-policies" + + +def headline(counts, verdict_value): + if verdict_value == "errored": + return "Tirith could not evaluate policies" + if verdict_value == "no-policies": + return "Tirith — no policies in scope for this workflow" + + parts = [] + for key, label in ((FAIL, "failed"), (APPROVAL_REQUIRED, "need approval"), (WARN, "warned")): + if counts.get(key): + parts.append(f"{counts[key]} {label}") + if counts.get(PASS): + parts.append(f"{counts[PASS]} passed") + if counts.get("SKIPPED"): + parts.append(f"{counts['SKIPPED']} skipped") + return "Tirith — " + (", ".join(parts) if parts else "nothing evaluated") + + +def _short_commit(commit): + """ + Seven characters, the length git itself abbreviates to. + + Anything that is not a hex sha is passed through untouched -- a tag or a branch name is more + useful whole, and truncating one would produce something that looks like a sha and is not. + """ + text = str(commit).strip() + if len(text) > 7 and all(c in "0123456789abcdefABCDEF" for c in text): + return text[:7] + return text + + +def render_cost(breakdown): + """ + One line of cost, for the pull-request comment. + + Rendered even when the estimate is zero or failed -- silence would be indistinguishable from + "this change costs nothing", and those are very different things to tell a reviewer. + Returns [] only when no estimate was attempted at all. + """ + if not isinstance(breakdown, dict) or not breakdown: + return [] + + if breakdown.get("error"): + return ["", "💵 Cost estimate unavailable for this plan."] + + currency = breakdown.get("currency") or "USD" + monthly = breakdown.get("totalMonthlyCost") + diff = breakdown.get("diffTotalMonthlyCost") + + if monthly is None: + return [] + + try: + monthly_text = f"{float(monthly):,.2f}" + except (TypeError, ValueError): + monthly_text = str(monthly) + + line = f"💵 Estimated monthly cost: **{_html(monthly_text)} {_html(currency)}**" + + # Infracost fills the diff from the plan's prior state, so it is the number a reviewer of a + # change actually wants. Only shown when it is non-zero and distinguishable from the total. + try: + delta = float(diff) + except (TypeError, ValueError): + delta = None + if delta: + line += f" ({'+' if delta > 0 else '−'}{abs(delta):,.2f} from this change)" + + return ["", f"{line}"] + + +def render_markdown( + policy_results, run_status, run_url, marker=None, limit=COMMENT_LIMIT, cost_breakdown=None, commit=None +): + """ + Render the results as markdown, truncating detail before the summary table. + + `marker` is an opaque first line the caller can use to find this document again -- GitHub's + sticky-comment marker, for instance. Kept as a parameter rather than built here so this module + stays VCS-agnostic. + + `commit` is the revision these findings describe. It matters because the comment is *edited in + place* across runs: without it a reader has no way to tell whether the verdict they are looking + at is about the head of the branch or about a push from an hour ago. Rendered here rather than + appended by the caller so the check-run summary and the job summary carry it too. + """ + counts, findings = summarize(policy_results) + verdict_value = verdict(counts, run_status) + + header = ([marker, ""] if marker else []) + [ + f"## 🛡️ {headline(counts, verdict_value)}", + "", + ] + if commit: + header += [f"Scanned commit {_html(_short_commit(commit))}", ""] + + if verdict_value == "errored": + # Two different reasons land here, and saying the wrong one is worse than saying nothing: + # a run that produced NOTHING, and a run whose results included one this tool cannot read. + # The second renders a populated table, under which "without producing policy results" + # reads as a plain contradiction. + if counts.get(UNKNOWN): + header += [ + f"{counts[UNKNOWN]} policy result(s) could not be read, so this run has no verdict.", + "This is reported as a failure rather than a pass: partial results are not a clean bill of health.", + "", + ] + else: + header += [ + f"The workflow run finished as {_code(run_status)} without producing policy results.", + "This is reported as a failure rather than a pass: no verdict is not the same as a clean one.", + "", + ] + + table = _render_table(findings) + # Ahead of the footer so the cost sits directly under the findings, and outside the truncation + # path below -- a long findings list must not push the cost line out of the comment. + cost = render_cost(cost_breakdown) + footer = cost + _render_footer(counts, run_url) + + detail_sections = [_render_detail(f) for f in findings if f["result"] in (FAIL, APPROVAL_REQUIRED, WARN, UNKNOWN)] + + body = "\n".join(header + table + detail_sections + footer) + if len(body) <= limit: + return body + + # Drop detail sections from the end until it fits, keeping the summary table intact -- the + # table is the part a reviewer scans first. + kept = list(detail_sections) + while kept and len(body) > limit: + kept.pop() + omitted = len(detail_sections) - len(kept) + note = [f"", f"_… and {omitted} more finding(s). See the full run in StackGuardian._", ""] + body = "\n".join(header + table + kept + note + footer) + + if len(body) > limit: + # Even the table is too large; truncate hard rather than risk a 422. + body = body[: limit - 200] + "\n\n_… truncated. See the full run in StackGuardian._\n" + + return body + + +def _code(value, in_table=False): + r""" + Render an untrusted value as an inline code span that cannot be closed from inside it. + + A penetration test found the report spoofable: every plan- and policy-derived string was + interpolated raw or wrapped in a single backtick, and a backtick *in the value* closes that span so + the remainder renders as markdown and HTML. A pull-request author controls the terraform a plan is + made from, so they controlled the gate report a reviewer reads -- a fake "all policies passed" + banner, a stray `
` collapsing the real findings, or a link whose text says one domain and + whose href says another. The verdict and exit code were never affected; the report was. + + Escaping the dangerous characters was the other option and is worse here. The engine deliberately + puts backticks in its own messages (`json_format_value` wraps every compared value in one), so + escaping them would put visible backslashes through every finding a reviewer reads, and it only + holds while the list of dangerous characters stays complete. + + A code span is inert by construction instead: per CommonMark a span opened by N backticks contains + any run of fewer than N, so a fence one longer than the longest run inside the value cannot be + closed by it. Nothing inside is interpreted -- no HTML, no links, no emphasis -- with no list to + enumerate. + + Two things a fence does not fix, both handled here: + * a newline ends the span, and ends a table row with it, so newlines collapse to a space; + * a pipe splits a table cell even inside a code span, and GFM's documented remedy is `\|`, which + is the one escape that works in there. Only applied for table cells, since outside a table the + backslash would show. + """ + text = "" if value is None else str(value) + text = " ".join(text.split()) + if in_table: + text = text.replace("|", "\\|") + + longest = 0 + run = 0 + for character in text: + run = run + 1 if character == "`" else 0 + longest = max(longest, run) + + fence = "`" * (longest + 1) + # A span whose content starts or ends with a backtick needs padding, or the delimiters merge. + pad = " " if text.startswith("`") or text.endswith("`") else "" + return f"{fence}{pad}{text}{pad}{fence}" + + +def _html(value): + """ + Escape an untrusted value for interpolation into inline HTML. + + A code span is the wrong tool inside ``, ``, `` or an attribute: GFM does not + reliably render markdown inside inline HTML, so the span would show as literal backticks. What + matters in an HTML context is that the value cannot terminate the element or the attribute it sits + in, which is what escaping the four characters does. Backticks are harmless here -- there is no + span to close. + + Backticks are escaped too, which `html.escape` does not do. They cannot close a span here because + there is none -- but an *odd* one can OPEN a span that runs on and swallows the markdown after it, + so a value like ``cost-control` `` inside `` still distorts the report even with the tags + neutralised. Turning it into an entity leaves it visible and inert. + """ + escaped = html.escape("" if value is None else str(value), quote=True) + return escaped.replace("`", "`") + + +def _render_table(findings): + if not findings: + return [] + rows = [ + "| | Policy | Rule | Resource |", + "|---|---|---|---|", + ] + for finding in findings: + icon = _ICONS.get(finding["result"], "⚪") + resources = ", ".join(_code(r, in_table=True) for r in finding["resources"][:3]) or "—" + if len(finding["resources"]) > 3: + resources += f" _+{len(finding['resources']) - 3}_" + rows.append( + f"| {icon} | {_code(finding['policy_id'], in_table=True)} " + f"| {_code(finding['rule_name'], in_table=True)} | {resources} |" + ) + rows.append("") + return rows + + +def _render_detail(finding): + icon = _ICONS.get(finding["result"], "⚪") + lines = [ + "
", + f"{icon} {_html(finding['policy_id'])} › {_html(finding['rule_name'])}" "", + "", + ] + for message in finding["messages"][:20]: + lines.append(f"- {_code(message)}") + if len(finding["messages"]) > 20: + lines.append(f"- _… and {len(finding['messages']) - 20} more_") + if finding["resources"]: + lines += ["", "Resources:"] + [f"- {_code(r)}" for r in finding["resources"][:20]] + lines += ["", "
", ""] + return "\n".join(lines) + + +def _render_footer(counts, run_url): + bits = [] + if counts.get(PASS): + bits.append(f"✅ {counts[PASS]} passed") + if counts.get("SKIPPED"): + bits.append(f"⚪ {counts['SKIPPED']} skipped") + if run_url: + bits.append(f'View run in StackGuardian') + return ["", f"{' · '.join(bits)}"] if bits else [] + + +def strip_marker(body): + """Drop the marker line, for a rendering target that has no use for it.""" + return "\n".join(line for line in body.split("\n") if not line.startswith("[//]: <>")) diff --git a/src/tirith/prettyprinter.py b/src/tirith/prettyprinter.py index 4134ba74..599f4100 100644 --- a/src/tirith/prettyprinter.py +++ b/src/tirith/prettyprinter.py @@ -97,7 +97,7 @@ def pretty_print_result_dict(final_result_dict: Dict) -> None: print(f" {TermStyle.fail('FAILED')}") num_failed_checks += 1 - for result_num, result_dict in enumerate(check_dict["result"]): + for result_num, result_dict in enumerate(check_dict.get("result", [])): result_message = result_dict["message"] if result_dict["passed"]: print(TermStyle.green(f" {result_num+1}. PASSED: {result_message}")) diff --git a/src/tirith/status.py b/src/tirith/status.py index d7ee3217..b5605ae7 100644 --- a/src/tirith/status.py +++ b/src/tirith/status.py @@ -9,6 +9,11 @@ class ExitStatus(IntEnum): ERROR = 1 ERROR_TIMEOUT = 2 + # A policy said no, under `--fail-on-error`. Distinct from ERROR so a caller can + # tell "your infrastructure violates a policy" from "tirith could not reach the platform" -- + # the same distinction --fail-on-error exists to draw, one level up. + ERROR_POLICY_FAILED = 3 + # # 128+2 SIGINT ERROR_CTRL_C = 130 diff --git a/tests/cli/test_dispatch.py b/tests/cli/test_dispatch.py new file mode 100644 index 00000000..df0c012a --- /dev/null +++ b/tests/cli/test_dispatch.py @@ -0,0 +1,108 @@ +""" +Tests for subcommand dispatch. + +The local-evaluation surface is a contract: the platform and the workflow-step templates parse its +--json output, and tests/core/test_output_compatibility.py asserts that output byte-for-byte. +Adding `tirith platform` must leave it completely untouched, including its single-dash long +options, which argparse cannot express alongside a subparser. +""" + +import json +import os + +import pytest + +from tirith import cli +from tirith.status import ExitStatus + +FIXTURES = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "providers", "json") +POLICY = os.path.join(FIXTURES, "policy.json") +INPUT = os.path.join(FIXTURES, "input.json") + + +def test_legacy_invocation_still_works(capsys): + """The flat parser must keep working exactly as before, driven through main(args=...).""" + status = cli.main(["-policy-path", POLICY, "-input-path", INPUT, "--json"]) + + assert status == ExitStatus.SUCCESS + document = json.loads(capsys.readouterr().out) + assert "final_result" in document + assert "evaluators" in document + + +def test_main_honours_its_args_parameter(capsys): + """ + It did not before: parse_args() was called with no argument, so main(args=...) was ignored and + the CLI always read sys.argv. That made it untestable and undrivable from another program. + """ + status = cli.main(["-policy-path", POLICY, "-input-path", INPUT, "--json"]) + + assert status == ExitStatus.SUCCESS + assert capsys.readouterr().out.strip().startswith("{") + + +def test_no_arguments_prints_help(capsys): + """ + Pre-existing behaviour, asserted so the dispatcher does not change it: the sys.exit(0) is + caught by main's own SystemExit handler, which returns None for a zero code. __main__ treats + that as success. + """ + status = cli.main([]) + + assert not status + assert "usage" in capsys.readouterr().out.lower() + + +def test_platform_is_dispatched_to_the_subcommand(capsys): + """`platform` with no subcommand prints the platform help, not the local-evaluation help.""" + status = cli.main(["platform"]) + + assert status == ExitStatus.SUCCESS + assert "tirith platform" in capsys.readouterr().out + + +def test_platform_check_requires_credentials(capsys, monkeypatch): + monkeypatch.delenv("SG_API_TOKEN", raising=False) + monkeypatch.delenv("SG_ORG", raising=False) + + status = cli.main(["platform", "check", "--workflow-id", "wf", "--input-path", INPUT]) + + assert status == ExitStatus.ERROR + assert "--api-key" in capsys.readouterr().err + + +def test_platform_check_requires_a_document(capsys, monkeypatch): + monkeypatch.setenv("SG_API_TOKEN", "sgo_x") + monkeypatch.setenv("SG_ORG", "acme") + + status = cli.main(["platform", "check", "--workflow-id", "wf"]) + + assert status == ExitStatus.ERROR + assert "--input-path" in capsys.readouterr().err + + +def test_a_bare_word_is_not_mistaken_for_a_subcommand(capsys): + """ + Only names in SUBCOMMANDS dispatch; anything else goes to the flat parser. + + `check` in particular must stay out: making it a top-level verb would mean the policy *source* + depended on whether SG_API_TOKEN happened to be exported, so an ambient environment variable could + silently swap local policy files for an organization's enforced set. + """ + assert cli.SUBCOMMAND == "platform" + assert "platform" in cli.SUBCOMMANDS + assert "check" not in cli.SUBCOMMANDS + + +def test_there_is_exactly_one_subcommand_name(capsys): + """ + `platform` was briefly renamed to `remote` and then reverted. Neither direction kept an alias -- + nothing is released, so there was never a caller to keep working -- and this pins the outcome: one + name, and `remote` is not quietly still accepted. + """ + assert cli.SUBCOMMANDS == {"platform"} + + status = cli.main(["remote"]) + + assert status != ExitStatus.SUCCESS + assert "tirith platform" not in capsys.readouterr().out diff --git a/tests/cli/test_local_gating.py b/tests/cli/test_local_gating.py new file mode 100644 index 00000000..157943c3 --- /dev/null +++ b/tests/cli/test_local_gating.py @@ -0,0 +1,173 @@ +""" +The local surface can gate, opt-in, without breaking the callers that rely on it not gating. + +`tirith -policy-path … -input-path …` has always exited 0 whether the policy passed or failed. That +made it useless as a CI gate on its own -- the only way to get an exit code that meant something was +to talk to StackGuardian, which is a poor answer for the path most open-source users are on. + +`--fail-on-error` fixes it without changing anything by default. The default is asserted here as +carefully as the new behaviour is: flipping it would turn every existing green pipeline red on upgrade, +which is exactly the kind of change that gets a tool pinned forever. + +The interesting cases are the ones that are neither a pass nor a violation. `final_result` is +tri-state: True passed, False said no, and **None means nothing ran** -- every check skipped. None is +not a pass, and it is not a violation either, so it exits 1: 3 means the infrastructure violates a +policy, 1 means tirith could not tell you. + +The first attempt at this gated on `errors` and inverted both halves. `errors` looks like a +tool-failure signal and is not -- it also carries the informational "these ids are not defined and +have been removed" note, so a genuine violation whose expression contained a typo exited 1 while a +policy naming an unknown provider exited 3. Both directions are tested below. +""" + +import json +import os +import sys + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "src")) + +from tirith.cli import main +from tirith.status import ExitStatus + +POLICY = { + "meta": { + "id": "instance-type", + "name": "instance types are approved", + "required_provider": "stackguardian/terraform_plan", + "version": "v1", + }, + "evaluators": [ + { + "id": "ev", + "description": "instance_type must be t3.micro", + "condition": {"type": "Equals", "value": "t3.micro", "error_tolerance": 0}, + "provider_args": { + "operation_type": "attribute", + "terraform_resource_attribute": "instance_type", + "terraform_resource_type": "aws_instance", + }, + } + ], + "eval_expression": "ev", +} + + +def _plan(instance_type): + return { + "format_version": "1.2", + "terraform_version": "1.5.7", + "resource_changes": [ + { + "address": "aws_instance.app", + "mode": "managed", + "type": "aws_instance", + "name": "app", + "change": { + "actions": ["create"], + "before": None, + "after": {"instance_type": instance_type}, + "after_sensitive": {}, + }, + } + ], + } + + +def _write(tmp_path, policy, instance_type="m5.24xlarge"): + policy_path = tmp_path / "policy.json" + plan_path = tmp_path / "plan.json" + policy_path.write_text(json.dumps(policy)) + plan_path.write_text(json.dumps(_plan(instance_type))) + return ["-policy-path", str(policy_path), "-input-path", str(plan_path), "--json"] + + +def test_a_failing_policy_still_exits_zero_by_default(tmp_path): + """ + The compatibility guarantee. Anyone already running this in CI is relying on it, knowingly or not, + and a silent change would break their pipeline on an upgrade they did not ask for. + """ + assert main(_write(tmp_path, POLICY)) == ExitStatus.SUCCESS + + +def test_a_failing_policy_exits_three_with_fail_on_error(tmp_path): + assert main(_write(tmp_path, POLICY) + ["--fail-on-error"]) == ExitStatus.ERROR_POLICY_FAILED + + +def test_a_passing_policy_exits_zero_with_fail_on_error(tmp_path): + args = _write(tmp_path, POLICY, instance_type="t3.micro") + assert main(args + ["--fail-on-error"]) == ExitStatus.SUCCESS + + +def test_an_unsupported_operator_is_one_not_three(tmp_path): + """ + `&` is not an operator the evaluator implements. It raises, so this exits 1 through the exception + handler rather than through the verdict branch -- worth having as an end-to-end assertion, but it + does not exercise the tri-state logic. The two tests below do. + """ + broken = dict(POLICY, eval_expression="ev & nonexistent") + + assert main(_write(tmp_path, broken) + ["--fail-on-error"]) == ExitStatus.ERROR + + +def test_a_policy_that_checked_nothing_is_one_not_three(tmp_path): + """ + `final_result: None` -- every check skipped, because `error_tolerance` swallowed a provider that + found nothing. Nothing ran, so there is no verdict: not a pass, and not a violation either. + + This is the case the flag exists for. Reporting 0 would be a green gate over an empty check, and + reporting 3 would tell someone their infrastructure violates a policy that never looked at it. + """ + skipped = dict(POLICY) + skipped["evaluators"] = [ + dict( + POLICY["evaluators"][0], + condition={"type": "Equals", "value": "x", "error_tolerance": 2}, + provider_args={ + "operation_type": "attribute", + "terraform_resource_type": "aws_nonexistent", + "terraform_resource_attribute": "nope", + }, + ) + ] + + assert main(_write(tmp_path, skipped) + ["--fail-on-error"]) == ExitStatus.ERROR + + +def test_a_violation_is_three_even_when_the_expression_names_an_undefined_id(tmp_path): + """ + A regression test for an inversion this had shipped. + + An `eval_expression` mentioning an id that does not exist produces an *informational* note in + `errors` -- "the following evaluator ids are not defined and have been removed" -- alongside a + perfectly real verdict. Gating on `errors` therefore reported a genuine violation as a tool + failure, which is the more dangerous direction: a broken gate looks like an outage and gets + retried, or worse, ignored. + """ + typo = dict(POLICY, eval_expression="ev && nonexistent") + + assert main(_write(tmp_path, typo) + ["--fail-on-error"]) == ExitStatus.ERROR_POLICY_FAILED + + +def test_a_missing_variable_is_one_not_three(tmp_path): + """ + The other unevaluable shape, and it fails differently: this path returns errors and no + `final_result` key at all, so a check that only looked at `final_result` would read the absence as + falsy and report a violation. + """ + parameterised = dict(POLICY, eval_expression="ev") + parameterised["evaluators"] = [ + dict(POLICY["evaluators"][0], condition={"type": "Equals", "value": "{{ var.expected }}", "error_tolerance": 0}) + ] + + exit_status = main(_write(tmp_path, parameterised) + ["--fail-on-error"]) + + assert exit_status != ExitStatus.ERROR_POLICY_FAILED, "an unresolved variable is not a policy violation" + + +@pytest.mark.parametrize("status", [ExitStatus.SUCCESS, ExitStatus.ERROR, ExitStatus.ERROR_POLICY_FAILED]) +def test_the_codes_this_relies_on_are_distinct(status): + """Guards the premise: 0, 1 and 3 have to be three different numbers for any of this to mean anything.""" + others = {ExitStatus.SUCCESS, ExitStatus.ERROR, ExitStatus.ERROR_POLICY_FAILED} - {status} + assert status.value not in {other.value for other in others} diff --git a/tests/core/test_core.py b/tests/core/test_core.py index 3afdc41e..ec09ea3f 100644 --- a/tests/core/test_core.py +++ b/tests/core/test_core.py @@ -151,3 +151,67 @@ def test_generate_evaluator_result_multiple_resources_one_failing(): assert len(result["result"]) == 2 assert result["result"][0]["passed"] is True assert result["result"][1]["passed"] is False + + +@mark.passing +def test_generate_evaluator_result_unsupported_evaluator_populates_result(): + """ + An unsupported condition.type must still produce a "result" list. Consumers index into + it unconditionally, so an early return without it used to raise KeyError far from the cause. + """ + evaluator_obj = { + "id": "test_evaluator", + "provider_args": {"operation_type": "attribute", "key": "value"}, + "condition": {"type": "NotAnEvaluator", "value": True}, + } + + with patch("tirith.core.core.get_evaluator_inputs_from_provider_inputs", return_value=[{"value": "x"}]): + result = generate_evaluator_result(evaluator_obj, {}, "test_provider") + + assert result["passed"] is False + assert result["result"] == [{"passed": False, "message": "`NotAnEvaluator` is not a supported evaluator"}] + + +@mark.passing +def test_generate_evaluator_result_bare_provider_err_is_surfaced(): + """ + A provider that reports "err" without a ProviderError is a malformed provider call (bad + operation_type, missing arg), not a policy violation. The message must reach the output + instead of being dropped and None evaluated against the condition. + """ + evaluator_obj = { + "id": "test_evaluator", + "provider_args": {"operation_type": "gt_value", "key": "value"}, + "condition": {"type": "Equals", "value": "us-east-1"}, + } + + bare_err = {"value": None, "meta": None, "err": "operation_type: gt_value is not supported"} + + with patch("tirith.core.core.get_evaluator_inputs_from_provider_inputs", return_value=[bare_err]): + with patch("tirith.core.core.EVALUATORS_DICT", {"Equals": MockEvaluator}): + result = generate_evaluator_result(evaluator_obj, {}, "test_provider") + + assert result["passed"] is False + assert len(result["result"]) == 1 + assert result["result"][0]["passed"] is False + assert result["result"][0]["message"] == "operation_type: gt_value is not supported" + + +@mark.passing +def test_generate_evaluator_result_bare_provider_err_ignores_error_tolerance(): + """error_tolerance tolerates missing data; it must never mask a malformed provider call.""" + evaluator_obj = { + "id": "test_evaluator", + "provider_args": {"operation_type": "gt_value", "key": "value"}, + # A tolerance high enough to swallow every documented severity, including 99. + "condition": {"type": "Equals", "value": "us-east-1", "error_tolerance": 100}, + } + + bare_err = {"value": None, "meta": None, "err": "operation_type: gt_value is not supported"} + + with patch("tirith.core.core.get_evaluator_inputs_from_provider_inputs", return_value=[bare_err]): + with patch("tirith.core.core.EVALUATORS_DICT", {"Equals": MockEvaluator}): + result = generate_evaluator_result(evaluator_obj, {}, "test_provider") + + assert result["passed"] is False, "a malformed provider call must not be skipped" + assert result["result"][0]["passed"] is False diff --git a/tests/core/test_output_compatibility.py b/tests/core/test_output_compatibility.py new file mode 100644 index 00000000..4dc64546 --- /dev/null +++ b/tests/core/test_output_compatibility.py @@ -0,0 +1,121 @@ +""" +Guardrails on the shape of the result document. + +The StackGuardian platform and the workflow-step templates parse this output, so its shape is a +contract rather than an implementation detail. `test_legacy_json_output_is_byte_identical` holds +the line: the golden file was captured before the engine changes landed, so any drift in the +single-policy output is a regression until proven otherwise. +""" + +import json +import os + +from pytest import mark + +from tirith.core.core import start_policy_evaluation_from_dict + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +GOLDEN_PATH = os.path.join(REPO_ROOT, "tests", "golden", "json_policy_output.json") + + +@mark.passing +def test_legacy_json_output_is_byte_identical(): + with open(os.path.join(REPO_ROOT, "tests", "providers", "json", "policy.json")) as f: + policy = json.load(f) + with open(os.path.join(REPO_ROOT, "tests", "providers", "json", "input.json")) as f: + input_data = json.load(f) + + result = start_policy_evaluation_from_dict(policy, input_data) + + with open(GOLDEN_PATH) as f: + # The golden file was captured from the CLI, whose print() adds a trailing newline + # that json.dumps does not produce. + expected = f.read().rstrip("\n") + + # indent=3 matches what the CLI emits (cli.py), so the golden file doubles as a + # record of the exact bytes a --json consumer receives. + assert json.dumps(result, indent=3) == expected + + +@mark.passing +def test_meta_passthrough_omits_absent_keys(): + """A policy declaring no optional metadata must produce exactly the two original keys.""" + policy = { + "meta": {"version": "v1", "required_provider": "stackguardian/json"}, + "evaluators": [ + { + "id": "check0", + "provider_args": {"operation_type": "get_value", "key_path": "a"}, + "condition": {"type": "Equals", "value": 1}, + } + ], + "eval_expression": "check0", + } + + result = start_policy_evaluation_from_dict(policy, {"a": 1}) + + assert result["meta"] == {"version": "v1", "required_provider": "stackguardian/json"} + + +@mark.passing +def test_meta_passthrough_carries_declared_keys(): + policy = { + "meta": { + "version": "v1", + "required_provider": "stackguardian/json", + "id": "no-public-ingress", + "name": "No 0.0.0.0/0 ingress", + "description": "Public ingress is not permitted", + "severity": "HIGH", + "enforcement": "hard_mandatory", + "tags": ["cis", "network"], + "remediation": "Restrict the CIDR or use a security-group reference", + }, + "evaluators": [ + { + "id": "check0", + "provider_args": {"operation_type": "get_value", "key_path": "a"}, + "condition": {"type": "Equals", "value": 1}, + } + ], + "eval_expression": "check0", + } + + result = start_policy_evaluation_from_dict(policy, {"a": 1}) + + assert result["meta"]["id"] == "no-public-ingress" + assert result["meta"]["name"] == "No 0.0.0.0/0 ingress" + assert result["meta"]["severity"] == "HIGH" + assert result["meta"]["enforcement"] == "hard_mandatory" + assert result["meta"]["tags"] == ["cis", "network"] + assert result["meta"]["remediation"] == "Restrict the CIDR or use a security-group reference" + # The originals survive alongside the additions. + assert result["meta"]["version"] == "v1" + assert result["meta"]["required_provider"] == "stackguardian/json" + + +@mark.passing +def test_meta_passthrough_supports_variables(): + """ + Variable substitution already covers the whole meta dict, so the new fields get + {{ var.x }} support without any extra plumbing. This pins that behaviour. + """ + policy = { + "meta": { + "version": "v1", + "required_provider": "stackguardian/json", + "severity": "{{ var.sev }}", + }, + "evaluators": [ + { + "id": "check0", + "provider_args": {"operation_type": "get_value", "key_path": "a"}, + "condition": {"type": "Equals", "value": 1}, + } + ], + "eval_expression": "check0", + } + + result = start_policy_evaluation_from_dict(policy, {"a": 1}, {"sev": "CRITICAL"}) + + assert result["meta"]["severity"] == "CRITICAL" diff --git a/tests/core/test_policy_parameterization.py b/tests/core/test_policy_parameterization.py index db9fcc04..08a55682 100644 --- a/tests/core/test_policy_parameterization.py +++ b/tests/core/test_policy_parameterization.py @@ -48,6 +48,56 @@ def test_not_found_variable(processed_policy): assert processed_policy[1] == ["key_path"] +def test_caller_policy_is_not_mutated(): + """Substitution must not write through to the caller's dict.""" + policy = { + "meta": {"version": "", "required_provider": "{{var.provider}}"}, + "evaluators": [ + { + "id": "check0", + "provider_args": {"operation_type": "get_value", "key_path": "a.b"}, + "condition": {"type": "Equals", "value": "{{var.expected}}"}, + } + ], + "eval_expression": "check0", + } + + replaced, not_found = get_policy_with_vars_replaced(policy, {"provider": "stackguardian/json", "expected": "yes"}) + + assert not_found == [] + # The copy carries the substituted values ... + assert replaced["meta"]["required_provider"] == "stackguardian/json" + assert replaced["evaluators"][0]["condition"]["value"] == "yes" + # ... while the original still carries the placeholders. + assert policy["meta"]["required_provider"] == "{{var.provider}}" + assert policy["evaluators"][0]["condition"]["value"] == "{{var.expected}}" + + +def test_same_policy_reused_with_different_vars(): + """ + A policy dict evaluated twice with different vars must not leak values between runs. + This is the multi-policy / retry case: without a deep copy the second call sees the + first call's substitutions already baked in and reports nothing to substitute. + """ + policy = { + "meta": {"version": "", "required_provider": "stackguardian/json"}, + "evaluators": [ + { + "id": "check0", + "provider_args": {"operation_type": "get_value", "key_path": "{{var.path}}"}, + "condition": {"type": "Equals", "value": True}, + } + ], + "eval_expression": "check0", + } + + first, _ = get_policy_with_vars_replaced(policy, {"path": "first.path"}) + second, _ = get_policy_with_vars_replaced(policy, {"path": "second.path"}) + + assert first["evaluators"][0]["provider_args"]["key_path"] == "first.path" + assert second["evaluators"][0]["provider_args"]["key_path"] == "second.path" + + # TODO: Create testcases for: # - test inline vars precendece over var files # - test undefined vars diff --git a/tests/golden/json_policy_output.json b/tests/golden/json_policy_output.json new file mode 100644 index 00000000..d0afad49 --- /dev/null +++ b/tests/golden/json_policy_output.json @@ -0,0 +1,87 @@ +{ + "meta": { + "version": "v1", + "required_provider": "stackguardian/json" + }, + "final_result": true, + "evaluators": [ + { + "id": "check0", + "passed": null, + "result": [ + { + "message": "key_path: `z.b` is not found (severity: 2)", + "passed": null + } + ], + "description": null + }, + { + "id": "check1", + "passed": true, + "result": [ + { + "passed": true, + "message": "`1` is less than equal to `1`", + "meta": null + } + ], + "description": null + }, + { + "id": "check2", + "passed": true, + "result": [ + { + "passed": true, + "message": "Found `\"aa\"` inside `[\"aa\", \"bb\"]`", + "meta": null + } + ], + "description": null + }, + { + "id": "check3", + "passed": true, + "result": [ + { + "passed": true, + "message": "`\"3\"` is equal to `\"3\"`", + "meta": null + } + ], + "description": null + }, + { + "id": "check4", + "passed": true, + "result": [ + { + "passed": true, + "message": "`\"value1\"` is equal to `\"value1\"`", + "meta": null + }, + { + "passed": true, + "message": "`\"value1\"` is equal to `\"value1\"`", + "meta": null + } + ], + "description": null + }, + { + "id": "check5", + "passed": true, + "result": [ + { + "passed": true, + "message": "`{\"e\": {\"f\": \"3\"}}` is equal to `{\"e\": {\"f\": \"3\"}}`", + "meta": null + } + ], + "description": null + } + ], + "errors": [], + "eval_expression": "check1 && check2 && check3 && check4 && check5" +} diff --git a/tests/platform/test_archive.py b/tests/platform/test_archive.py new file mode 100644 index 00000000..b6681a23 --- /dev/null +++ b/tests/platform/test_archive.py @@ -0,0 +1,476 @@ +""" +Tests for the project archive. + +The assertions that matter read the bytes *inside the built tarball*, not the objects handed to +pack(). That distinction is the whole point: a previous iteration of this code masked a plan +correctly in memory and still shipped the plaintext, because the secret lived in a second place +nobody had looked at. Asserting on the input would have passed. +""" + +import io +import json +import os +import tarfile + +import pytest + +from tirith.platform import archive + +SECRET = "hunter2-this-must-never-leave-the-runner" + + +def members(archive_bytes): + with tarfile.open(fileobj=io.BytesIO(archive_bytes), mode="r:gz") as tar: + return sorted(tar.getnames()) + + +def read_member(archive_bytes, name): + with tarfile.open(fileobj=io.BytesIO(archive_bytes), mode="r:gz") as tar: + return tar.extractfile(name).read() + + +def raw_bytes(archive_bytes): + """Everything in the archive, decompressed, as one blob -- for leak assertions.""" + blob = b"" + with tarfile.open(fileobj=io.BytesIO(archive_bytes), mode="r:gz") as tar: + for member in tar.getmembers(): + blob += member.name.encode() + if member.isfile(): + blob += tar.extractfile(member).read() + return blob + + +# --- documents --------------------------------------------------------------------------------- + + +def test_documents_land_at_the_fixed_names_the_step_looks_for(tmp_path): + body, _manifest = archive.pack(source_dir=None, plan={"a": 1}, state={"b": 2}, infracost={"c": 3}) + + assert members(body) == ["infracost.json", "plan.json", "tfstate.json"] + assert json.loads(read_member(body, "plan.json")) == {"a": 1} + + +def test_absent_documents_are_simply_not_written(): + body, _manifest = archive.pack(source_dir=None, state={"version": 4}) + + assert members(body) == ["tfstate.json"] + + +def test_masked_document_wins_over_a_stale_file_on_disk(tmp_path): + """ + The dangerous ordering: a plan.json left in the working directory from an earlier run would + otherwise be packed *and* the masked one written, shipping both. + """ + (tmp_path / "plan.json").write_text(json.dumps({"leaked": SECRET})) + + body, _manifest = archive.pack(source_dir=str(tmp_path), plan={"masked": "__SG_REDACTED__"}) + + assert json.loads(read_member(body, "plan.json")) == {"masked": "__SG_REDACTED__"} + assert SECRET.encode() not in raw_bytes(body) + + +@pytest.mark.parametrize("name", ["plan.json", "tfstate.json", "infracost.json"]) +def test_reserved_names_on_disk_are_never_packed(tmp_path, name): + """ + The leak this closes: `terraform state pull > state.json` is the documented way to produce a + state file, so one routinely sits in the working directory -- raw and unmasked. Packing the + source tree naively shipped it in full, right next to the masked copy. + + These names are only ever written by pack() from an already-masked object. A caller who wants + the file evaluated passes --state-path / --input-path, which masks it first. + """ + (tmp_path / name).write_text(json.dumps({"outputs": {"db": {"value": SECRET}}})) + (tmp_path / "main.tf").write_text("") + + body, _manifest = archive.pack(source_dir=str(tmp_path), plan={"masked": True}) + + assert SECRET.encode() not in raw_bytes(body) + assert members(body) == ["code/main.tf", "plan.json"] + + +@pytest.mark.parametrize("name", ["tfplan.json", "state.json", "terraform.plan.json"]) +def test_the_file_a_document_was_read_from_is_never_packed(tmp_path, name): + """ + Reserving only the three names pack() writes was not enough. The input is routinely called + something else -- `tfplan.json` is the second name discovery accepts, and + `terraform state pull > state.json` is the documented way to produce state -- so the source walk + shipped the unmasked original one filename away from the masked copy. + """ + (tmp_path / name).write_text(json.dumps({"outputs": {"db": {"value": SECRET}}})) + (tmp_path / "main.tf").write_text("") + + body, _manifest = archive.pack( + source_dir=str(tmp_path), + plan={"masked": True}, + document_sources=(str(tmp_path / name),), + ) + + assert SECRET.encode() not in raw_bytes(body) + assert members(body) == ["code/main.tf", "plan.json"] + + +def test_the_binary_plan_is_never_packed(tmp_path): + """ + A binary plan embeds the prior state, so it carries every attribute of every existing resource + in plaintext -- worse than a raw state file, and it matches none of the *.tfstate patterns. + --plan-file converts and masks it in memory, which the source walk then undid. + """ + (tmp_path / "tfplan").write_bytes(b"\x1f\x8b binary plan " + SECRET.encode()) + (tmp_path / "prod.tfplan").write_bytes(SECRET.encode()) + (tmp_path / "main.tf").write_text("") + + body, _manifest = archive.pack(source_dir=str(tmp_path), plan={"masked": True}) + + assert SECRET.encode() not in raw_bytes(body) + assert members(body) == ["code/main.tf", "plan.json"] + + +def test_a_document_source_outside_the_tree_excludes_nothing(tmp_path): + """ + An out-of-tree path cannot collide with a member name, so it must not be reduced to a bare + basename -- doing so would silently drop an unrelated same-named file from the archive. + """ + outside = tmp_path / "elsewhere" + outside.mkdir() + (outside / "main.tf").write_text("") + source = tmp_path / "src" + source.mkdir() + (source / "main.tf").write_text("resource {}") + + body, _manifest = archive.pack( + source_dir=str(source), + plan={"masked": True}, + document_sources=(str(outside / "main.tf"),), + ) + + assert members(body) == ["code/main.tf", "plan.json"] + assert read_member(body, "code/main.tf") == b"resource {}" + + +def test_masked_document_is_what_gets_written(tmp_path): + """The counterpart: a supplied document really does reach the archive.""" + (tmp_path / "tfstate.json").write_text(json.dumps({"secret": SECRET})) + + body, _manifest = archive.pack(source_dir=str(tmp_path), state={"masked": True}) + + assert json.loads(read_member(body, "tfstate.json")) == {"masked": True} + assert SECRET.encode() not in raw_bytes(body) + + +# --- exclusions -------------------------------------------------------------------------------- + + +def test_terraform_provider_cache_is_excluded(tmp_path): + """A provider cache is routinely hundreds of MB; shipping it would make every run unusable.""" + provider = tmp_path / ".terraform" / "providers" / "registry.terraform.io" + provider.mkdir(parents=True) + (provider / "terraform-provider-aws").write_bytes(b"x" * 1024) + (tmp_path / "main.tf").write_text('resource "null_resource" "a" {}') + + body, manifest = archive.pack(source_dir=str(tmp_path)) + + assert members(body) == ["code/main.tf"] + assert manifest["skipped"] >= 1 + + +def test_git_directory_is_excluded(tmp_path): + """.git carries full history, so anything ever committed would ship.""" + (tmp_path / ".git").mkdir() + (tmp_path / ".git" / "config").write_text(f"token = {SECRET}") + (tmp_path / "main.tf").write_text("") + + body, _manifest = archive.pack(source_dir=str(tmp_path)) + + assert members(body) == ["code/main.tf"] + assert SECRET.encode() not in raw_bytes(body) + + +@pytest.mark.parametrize("name", ["terraform.tfstate", "terraform.tfstate.backup", "prod.tfstate"]) +def test_raw_state_files_are_excluded(tmp_path, name): + """ + Raw state is unmasked by definition. Left in, it would travel next to the masked copy and + undo the masking entirely. + """ + (tmp_path / name).write_text(json.dumps({"outputs": {"db": {"value": SECRET}}})) + (tmp_path / "main.tf").write_text("") + + body, _manifest = archive.pack(source_dir=str(tmp_path)) + + assert f"code/{name}" not in members(body) + assert SECRET.encode() not in raw_bytes(body) + + +def test_gitignore_is_honoured(tmp_path): + (tmp_path / ".gitignore").write_text("secrets.auto.tfvars\nbuild/\n") + (tmp_path / "secrets.auto.tfvars").write_text(f'password = "{SECRET}"') + (tmp_path / "build").mkdir() + (tmp_path / "build" / "out.bin").write_text("junk") + (tmp_path / "main.tf").write_text("") + + body, _manifest = archive.pack(source_dir=str(tmp_path)) + + assert "code/secrets.auto.tfvars" not in members(body) + assert "code/build/out.bin" not in members(body) + assert SECRET.encode() not in raw_bytes(body) + + +def test_gitignore_can_be_turned_off(tmp_path): + (tmp_path / ".gitignore").write_text("keep-me.tf\n") + (tmp_path / "keep-me.tf").write_text("") + + body, _manifest = archive.pack(source_dir=str(tmp_path), respect_gitignore=False) + + assert "code/keep-me.tf" in members(body) + + +def test_extra_excludes_are_applied(tmp_path): + (tmp_path / "big.zip").write_text("junk") + (tmp_path / "main.tf").write_text("") + + body, _manifest = archive.pack(source_dir=str(tmp_path), extra_excludes=("*.zip",)) + + assert members(body) == ["code/main.tf"] + + +def test_lock_file_is_kept(tmp_path): + """It pins provider versions, is small, and the run controller's init wants it.""" + (tmp_path / ".terraform.lock.hcl").write_text("provider ...") + + body, _manifest = archive.pack(source_dir=str(tmp_path)) + + assert "code/.terraform.lock.hcl" in members(body) + + +def test_symlinks_are_skipped(tmp_path): + """A symlink out of the tree either breaks on extraction or smuggles a file in.""" + outside = tmp_path.parent / "outside.txt" + outside.write_text(SECRET) + source = tmp_path / "src" + source.mkdir() + (source / "main.tf").write_text("") + os.symlink(str(outside), str(source / "link.txt")) + + body, _manifest = archive.pack(source_dir=str(source)) + + assert members(body) == ["code/main.tf"] + assert SECRET.encode() not in raw_bytes(body) + + +# --- structure --------------------------------------------------------------------------------- + + +def test_nested_directories_keep_their_relative_paths(tmp_path): + (tmp_path / "modules" / "vpc").mkdir(parents=True) + (tmp_path / "modules" / "vpc" / "main.tf").write_text("") + (tmp_path / "main.tf").write_text("") + + body, _manifest = archive.pack(source_dir=str(tmp_path)) + + assert "code/modules/vpc/main.tf" in members(body) + + +def test_no_source_dir_is_allowed(): + """--no-source: send only the documents.""" + body, manifest = archive.pack(source_dir=None, plan={"a": 1}) + + assert members(body) == ["plan.json"] + assert manifest["files"] == 0 + + +def test_missing_source_dir_is_an_error(tmp_path): + with pytest.raises(archive.ArchiveError): + archive.pack(source_dir=str(tmp_path / "does-not-exist")) + + +def test_oversized_archive_is_refused(tmp_path, monkeypatch): + """ + Failing loudly beats a five-minute upload that times out the run. A runaway archive is nearly + always an exclusion that did not fire. + """ + monkeypatch.setattr(archive, "MAX_ARCHIVE_BYTES", 512) + (tmp_path / "big.tf").write_text("resource {}\n" * 20000) + + with pytest.raises(archive.ArchiveError, match="limit"): + archive.pack(source_dir=str(tmp_path)) + + +def test_manifest_reports_what_went_in(tmp_path): + (tmp_path / "main.tf").write_text("") + (tmp_path / ".terraform").mkdir() + (tmp_path / ".terraform" / "x").write_text("") + + _body, manifest = archive.pack(source_dir=str(tmp_path), plan={"a": 1}) + + assert manifest["files"] == 1 + assert manifest["documents"] == ["plan.json"] + assert manifest["skipped"] >= 1 + assert manifest["bytes"] > 0 + + +def test_the_binary_plan_that_plan_file_read_is_never_packed(tmp_path): + """ + --plan-file converts the binary plan in memory precisely so nothing unmasked touches the disk -- + but the binary plan is already on disk, and it embeds the prior state: every attribute of every + existing resource. The `tfplan` name patterns only cover the spellings the README uses, and + `terraform plan -out=plan.out` is at least as common. + """ + (tmp_path / "plan.out").write_bytes(b"binary plan " + SECRET.encode()) + (tmp_path / "main.tf").write_text("") + + body, _manifest = archive.pack( + source_dir=str(tmp_path), + plan={"masked": True}, + document_sources=(str(tmp_path / "plan.out"),), + ) + + assert SECRET.encode() not in raw_bytes(body) + assert members(body) == ["code/main.tf", "plan.json"] + + +# --- layout: code/ is a prefix, the root belongs to the documents ------------------------------- + + +def test_the_source_lives_under_the_code_prefix(tmp_path): + """ + The layout is a contract for whatever reads the bundle: source under `code/`, documents at the + root, and `code/x` maps back to `/x`. + """ + (tmp_path / "main.tf").write_text("") + (tmp_path / "modules" / "vpc").mkdir(parents=True) + (tmp_path / "modules" / "vpc" / "main.tf").write_text("") + + body, _manifest = archive.pack(source_dir=str(tmp_path), plan={"masked": True}) + + assert members(body) == ["code/main.tf", "code/modules/vpc/main.tf", "plan.json"] + + +def test_the_documents_are_at_the_archive_root_and_never_under_a_prefix(tmp_path): + """ + The one layout mistake that would not fail loudly. + + The step finds its inputs with a flat join onto the extraction directory and treats absence as + normal (`_discover_document` returns None). So a document moved under `code/` -- or under any + prefix -- would not raise: every policy would come back unevaluated and the run would report as + passed-with-warnings. Nothing downstream distinguishes that from a genuinely clean plan, which is + why this is asserted here rather than trusted. + """ + (tmp_path / "main.tf").write_text("") + + body, _manifest = archive.pack( + source_dir=str(tmp_path), + plan={"masked": True}, + state={"masked": True}, + infracost={"masked": True}, + ) + + for document in (archive.PLAN_DOCUMENT, archive.STATE_DOCUMENT, archive.INFRACOST_DOCUMENT): + assert document in members(body), f"{document} must be at the archive root" + assert not [name for name in members(body) if name.endswith(f"/{archive.PLAN_DOCUMENT}")] + + +def test_a_committed_document_name_is_still_skipped_under_the_prefix(tmp_path): + """ + The reservation is a leak guard, and it stopped being self-evident when the prefix arrived. + + Under the old flat layout a committed `tfstate.json` would have collided with the masked one, so + skipping it looked obviously necessary. `code/tfstate.json` cannot collide with anything -- and is + still raw, unmasked state, which is the actual reason for the skip. Deleting it because "the + collision is impossible now" is the mistake this test exists to catch. + """ + (tmp_path / "tfstate.json").write_text(json.dumps({"secret": SECRET})) + (tmp_path / "main.tf").write_text("") + + body, _manifest = archive.pack(source_dir=str(tmp_path), state={"masked": True}) + + assert "code/tfstate.json" not in members(body) + assert SECRET.encode() not in raw_bytes(body) + + +def test_metadata_is_absent_unless_asked_for(tmp_path): + """A caller that supplies no metadata gets no member, so old bundles stay describable as such.""" + (tmp_path / "main.tf").write_text("") + + body, _manifest = archive.pack(source_dir=str(tmp_path)) + + assert archive.METADATA_DOCUMENT not in members(body) + + +# --- metadata.json: what pack() observes, as opposed to what it was told ------------------------ + + +def _metadata(archive_bytes): + return json.loads(read_member(archive_bytes, archive.METADATA_DOCUMENT)) + + +def test_metadata_records_what_was_actually_packed(tmp_path): + """ + The counts come from the walk, not from the caller. "Claims code, packed nothing" is only + detectable because this half of the document is produced here. + """ + (tmp_path / "main.tf").write_text("") + (tmp_path / "notes.txt").write_text("") + + body, _manifest = archive.pack( + source_dir=str(tmp_path), + plan={"masked": True}, + metadata={"schema_version": 1, "code": {"repo_path": "infra/prod", "repo_path_from": "flag"}}, + ) + + code = _metadata(body)["code"] + assert code["present"] is True + assert code["prefix"] == "code/" + assert code["files"] == 2 + assert code["repo_path"] == "infra/prod" + assert _metadata(body)["documents"] == {"plan": "plan.json", "state": None, "infracost": None} + + +def test_metadata_cannot_claim_code_the_archive_does_not_carry(tmp_path): + """ + `present` means "there are members under the prefix", not "a source directory was requested". + + Nothing writes an explicit directory entry, so a tree whose every file was excluded leaves no + `code/` in the tar at all. A consumer comparing the tar against the metadata must never find them + disagreeing, so the flag is derived from the count rather than from the request. + """ + (tmp_path / "everything.tfstate").write_text("raw state") + + body, _manifest = archive.pack( + source_dir=str(tmp_path), + plan={"masked": True}, + metadata={"schema_version": 1, "code": {"repo_path": "infra", "repo_path_from": "flag"}}, + ) + + code = _metadata(body)["code"] + assert code["present"] is False + assert code["prefix"] is None + assert code["files"] == 0 + # And the path is withdrawn: there is nothing for it to describe. + assert code["repo_path"] is None + assert code["absent_reason"] == "empty_after_excludes" + assert not [name for name in members(body) if name.startswith("code/")] + + +def test_metadata_says_why_no_source_was_requested(tmp_path): + """ + A documents-only bundle has to distinguish "none wanted" from "dropped for size", or a consumer + cannot tell a deliberate configuration from a truncated one. + """ + body, _manifest = archive.pack( + source_dir=None, + plan={"masked": True}, + metadata={"schema_version": 1, "code": {"absent_reason": "not_requested"}}, + ) + + code = _metadata(body)["code"] + assert code["present"] is False + assert code["absent_reason"] == "not_requested" + + +def test_metadata_does_not_mutate_the_caller_dict(tmp_path): + """The retry path re-packs with a modified copy; mutating the original would corrupt it.""" + (tmp_path / "main.tf").write_text("") + supplied = {"schema_version": 1, "code": {"repo_path": "infra"}} + + archive.pack(source_dir=str(tmp_path), metadata=supplied) + + assert supplied == {"schema_version": 1, "code": {"repo_path": "infra"}} diff --git a/tests/platform/test_check.py b/tests/platform/test_check.py new file mode 100644 index 00000000..9abe62b8 --- /dev/null +++ b/tests/platform/test_check.py @@ -0,0 +1,503 @@ +""" +Tests for the check orchestration. + +Focused on `upload_state_document`, because that is the one place in this codebase that can overwrite +a customer's live terraform state. `artifacts/tfstate.json` is not just a name we picked: the +managed-state backend writes it, state locking keys on the literal basename, and the state-backends +view lists it. Writing a *masked* document there for a workflow that manages its own state would be +data loss, so the guard is asserted rather than assumed. +""" + +import json +import os +import sys + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "src")) + +from tirith.platform import check +from tirith.platform.client import SGError + + +class FakeClient: + def __init__(self, managed=False, fail=False): + self.managed = managed + self.fail = fail + self.uploads = [] + + def manages_terraform_state(self, wfgrp, workflow_id): + return self.managed + + def upload_file(self, wfgrp, workflow_id, filename, folder, content, content_type=None): + if self.fail: + raise SGError("presigned URL expired") + self.uploads.append( + { + "filename": filename, + "folder": folder, + "content": content, + "content_type": content_type, + } + ) + return f"orgs/acme/wfs/K/artifacts/{filename}" + + +class Opts: + workflow_group = "default" + workflow_id = "wf" + + +STATE = { + "version": 4, + "resources": [{"type": "aws_s3_bucket", "instances": [{"attributes": {"b": "__SG_REDACTED__"}}]}], +} + + +def test_the_state_is_published_as_tfstate_json(): + client = FakeClient(managed=False) + + check.upload_state_document(client, Opts(), STATE) + + assert len(client.uploads) == 1 + upload = client.uploads[0] + assert upload["filename"] == "tfstate.json" + # The artifacts root, not a subfolder: that is the key the platform reads. + assert upload["folder"] is None + assert upload["content_type"] == "application/json" + assert json.loads(upload["content"].decode()) == STATE + + +def test_the_state_is_not_written_over_a_managed_state_workflow(capsys): + """The data-loss guard. That object is the live state for such a workflow.""" + client = FakeClient(managed=True) + + check.upload_state_document(client, Opts(), STATE) + + assert client.uploads == [] + warning = capsys.readouterr().err + assert "manages its own terraform state" in warning + # And it says the state is still evaluated, so the skip does not read as a lost check. + assert "still evaluated" in warning + + +def test_a_failed_publish_is_a_warning_not_a_failure(capsys): + """ + The verdict does not depend on this upload. A run whose policies evaluated perfectly well must not + go red because a best-effort convenience copy could not be written. + """ + client = FakeClient(managed=False, fail=True) + + check.upload_state_document(client, Opts(), STATE) + + assert "could not publish tfstate.json" in capsys.readouterr().err + + +def test_the_published_state_is_flagged_as_masked(capsys): + """ + A file at the canonical state key that looks like state but is full of __SG_REDACTED__ is a + footgun for whoever downloads it next, so the log says so. + """ + check.upload_state_document(FakeClient(managed=False), Opts(), STATE) + + assert "cannot be used to run terraform" in capsys.readouterr().err + + +def test_the_state_document_name_matches_the_one_inside_the_archive(): + """ + The step reads the archive copy to publish TfStateCleaned while the platform reads the uploaded + one. Two different names would be two sources of truth for the same thing. + """ + from tirith.platform import archive + + assert check.STATE_DOCUMENT_NAME == archive.STATE_DOCUMENT + + +# --- packing: the source is uploaded, but never at the cost of the gate -------------------------- +# +# The source tree is packed by default, so an exclusion that does not fire -- a committed vendor +# directory, a build output tree -- would otherwise turn a working policy check into a failed run. +# That trade is the wrong way round: the verdict gates the merge, the source is a convenience for +# whatever reads the bundle afterwards. + + +def _tree(tmp_path, extra_bytes=0): + source = tmp_path / "src" + source.mkdir() + (source / "main.tf").write_text('resource "null_resource" "a" {}\n') + if extra_bytes: + # Random, so gzip cannot make it disappear. + (source / "vendor.bin").write_bytes(os.urandom(extra_bytes)) + return str(source) + + +def test_the_source_is_packed_on_the_normal_path(tmp_path): + archive_bytes, manifest, skipped = check.pack_documents(_tree(tmp_path), {"masked": True}, None, None) + + assert manifest["files"] == 1 + assert manifest["documents"] == ["plan.json"] + assert skipped is None + assert archive_bytes + + +def test_an_oversized_source_tree_degrades_to_documents_only(tmp_path, monkeypatch, capsys): + monkeypatch.setattr(check.archive, "MAX_ARCHIVE_BYTES", 50 * 1024) + + archive_bytes, manifest, skipped = check.pack_documents( + _tree(tmp_path, extra_bytes=200_000), {"masked": True}, None, None + ) + + # The documents still go, so the policies still run. + assert manifest["documents"] == ["plan.json"] + assert manifest["files"] == 0 + # And the caller can tell that the bundle has no code in it. + assert skipped and "over the" in skipped + + warning = capsys.readouterr().err + assert "carries no code" in warning + assert "--source-dir" in warning + + +def test_an_oversized_documents_only_archive_still_fails(tmp_path, monkeypatch): + """ + Nothing left to drop. Degrading further would mean uploading an archive with no documents, which + is not a check at all -- so this stays fatal rather than becoming a silent pass. + """ + monkeypatch.setattr(check.archive, "MAX_ARCHIVE_BYTES", 50 * 1024) + + with pytest.raises(check.archive.ArchiveError): + check.pack_documents(None, {"blob": os.urandom(200_000).hex()}, None, None) + + +def test_the_size_message_is_readable_below_a_megabyte(monkeypatch): + """ + Integer MB division reported everything small as "0 MB over the 0 MB limit". That message is now + surfaced on a pull request, where it has to mean something. + """ + from tirith.platform.archive import _human_bytes + + assert _human_bytes(137 * 1024 * 1024) == "137.0 MB" + assert _human_bytes(300 * 1024) == "300.0 KB" + assert _human_bytes(512) == "512 bytes" + + +def test_the_policy_step_is_spliced_in_as_a_pre_plan_step(): + """ + The whole mechanism, and it uses only primitives the platform already had: core splices + `prePlanWfStepsConfig` ahead of `generate-terraform-plan`, and the step exits 12, which tells the + run controller to complete the run and skip everything after it. So core needs to know nothing + about this feature -- which is why there is no terraform action for it. + """ + config = check.terraform_config("1.5.7", None) + + steps = config["prePlanWfStepsConfig"] + assert len(steps) == 1 + assert steps[0]["name"] == check.POLICY_STEP_NAME + assert steps[0]["wfStepTemplateId"] == check.POLICY_STEP_TEMPLATE + # Every input the step needs travels in its own step input, not the terraform configuration. + assert steps[0]["wfStepInputData"]["schemaType"] == "FORM_JSONSCHEMA" + # A policy check writes no state, so it must not take a managed-state backend override. + assert config["managedTerraformState"] is False + # No stored input kind: routing is by which document is in the archive. + assert "policyInputKind" not in config + + +def test_a_step_template_override_is_honoured(): + config = check.terraform_config("1.5.7", "/demo-org/tirith-iac-governance:3") + + assert config["prePlanWfStepsConfig"][0]["wfStepTemplateId"] == "/demo-org/tirith-iac-governance:3" + + +# --- the bundle is named per commit, and per RUN ------------------------------------------------ +# +# A name shared by every run of the workflow is one two concurrent runs can overwrite, and the action +# derives a single workflow id per repository -- so two open pull requests, the ordinary case, would +# have one run evaluating the other's code and reporting the verdict as its own. Silently, on a merge +# gate. Naming it per commit removes the collision rather than detecting it afterwards. +# +# That is only possible because the name travels per RUN: core merges the run's TerraformConfig over +# the workflow's, so `prePlanWfStepsConfig` can differ every time. The workflow's stored copy is +# written once, at creation, and never updated. + + +def test_the_bundle_name_carries_the_commit(): + from tirith.platform.client import ARCHIVE_NAME_TEMPLATE + + name = ARCHIVE_NAME_TEMPLATE.format(sha="a1b2c3d", tag="plan") + + assert name == "tirith-bundle-a1b2c3d-plan.tar.gz" + # Two commits cannot collide, which is the entire point. + assert name != ARCHIVE_NAME_TEMPLATE.format(sha="9999999", tag="plan") + + +def test_the_bundle_name_survives_the_artifact_syncs_exclude_list(): + """ + The sync is the delivery mechanism, so a name matching any of its excludes would be dropped + silently and never reach the container. `__sg.`, which this name used to carry, is excluded + precisely so the old carrier stayed OUT of the sync -- exactly wrong now. + """ + import fnmatch + + from tirith.platform.client import ARCHIVE_NAME_TEMPLATE + + name = ARCHIVE_NAME_TEMPLATE.format(sha="a1b2c3d", tag="plan") + excluded = ("sg.*", "__sg.*", "*__sg.*", "*pci_*", "*_thrifty_*", "*_gdpr_*", "*compliance_raw*") + + for pattern in excluded: + assert not fnmatch.fnmatch(name, pattern), f"the bundle name matches the sync exclude {pattern!r}" + assert name != "tfstate.json", "that name is a managed-state workflow's live state" + + +def test_the_run_names_its_own_bundle(): + """ + The per-run half. `wfStepInputData` on the *workflow* is written once and never updated, so the + name has to be re-sent with each run for it to describe that run's commit. + """ + step = check.policy_step(None, "tirith-bundle-a1b2c3d-plan.tar.gz") + + assert step["wfStepInputData"]["data"]["bundlePath"] == "tirith-bundle-a1b2c3d-plan.tar.gz" + # Sent in full: core's merge is shallow, so supplying prePlanWfStepsConfig replaces the whole + # list and a partial entry would lose the template id the step runs from. + assert step["wfStepTemplateId"] == check.POLICY_STEP_TEMPLATE + assert step["name"] == check.POLICY_STEP_NAME + assert step["timeout"] == check.POLICY_STEP_TIMEOUT + + +def test_the_step_template_override_reaches_the_per_run_step(): + step = check.policy_step("/demo-org/tirith-iac-governance:3", "b.tar.gz") + + assert step["wfStepTemplateId"] == "/demo-org/tirith-iac-governance:3" + + +def test_the_run_tells_the_step_whether_state_is_managed(): + """ + The step writes the masked state to `artifacts/tfstate.json`, which for a managed-state workflow is + the LIVE state. It must be told, and told explicitly rather than left to a default: a missing key + that happens to mean "not managed" is one refactor away from meaning the opposite. + """ + step = check.policy_step(None, "tirith-bundle-a1b2c3d-plan.tar.gz") + + data = step["wfStepInputData"]["data"] + assert data["managedTerraformState"] is False + + +def test_the_workflow_never_takes_a_managed_state_backend(): + """And the claim the passthrough rests on: these workflows do not manage state in the first place.""" + config = check.terraform_config("1.5.7", None) + + assert config["managedTerraformState"] is False + + +# --- metadata.json: the provenance half ----------------------------------------------------------- +# +# Two shapes have to produce an honest document: a CI run, where the trigger payload carries the +# repository and commit, and a bare local invocation, where none of it exists. The local case is the +# one worth guarding -- the temptation is to fill the gaps from the environment, and a fabricated +# repository in a file that outlives the run is worse than a null. + + +class MetaOpts: + trigger_details = {"type": "cli"} + repo_url = None + repo_ref = None + repo_path = None + sha = None + source_dir = None + input_kind = "terraform_plan" + org = "acme" + workflow_group = "default" + workflow_id = "infra" + artifact_tag = "default" + + +def _opts(**overrides): + opts = MetaOpts() + for key, value in overrides.items(): + setattr(opts, key, value) + return opts + + +def test_a_local_run_states_it_is_local_rather_than_leaving_ci_to_be_inferred(): + metadata = check.build_metadata(_opts(), redactions=0) + + assert metadata["origin"] == {"kind": "local", "trigger_type": "cli", "ci_run_url": None} + # Nulls, not omissions, and nothing invented. + assert metadata["repository"]["provider"] == "unknown" + assert metadata["repository"]["url"] is None + assert metadata["repository"]["commit"] is None + assert metadata["repository"]["change_request"] is None + assert metadata["schema_version"] == check.METADATA_SCHEMA_VERSION + + +def test_a_ci_run_records_the_repository_and_the_change_request(): + opts = _opts( + trigger_details={ + "type": "tirith", + "repoHttpUrl": "https://github.com/acme/infra", + "headSha": "9f2c1ab5", + "ref": "feat/rds", + "prId": "412", + "eventSource": "https://github.com/acme/infra/pull/412", + "runUrl": "https://github.com/acme/infra/actions/runs/1", + } + ) + + metadata = check.build_metadata(opts, redactions=12) + + assert metadata["origin"]["kind"] == "ci" + assert metadata["repository"]["provider"] == "github" + assert metadata["repository"]["commit"] == "9f2c1ab5" + assert metadata["repository"]["change_request"]["id"] == "412" + assert metadata["masking"]["redactions"] == 12 + + +def test_a_credential_in_the_repo_url_never_reaches_the_metadata(): + """ + `https://x-access-token:ghs_…@github.com/…` is an ordinary value for a CI checkout to hold, and + GitLab's own CI_REPOSITORY_URL embeds a job token the same way. This file ships inside the bundle + and outlives the run, so a token written here is a token persisted in an artifact. + """ + import json as _json + + opts = _opts(repo_url="https://x-access-token:ghs_verysecret@github.com/acme/infra.git") + + metadata = check.build_metadata(opts, redactions=0) + + assert "ghs_verysecret" not in _json.dumps(metadata) + assert "x-access-token" not in _json.dumps(metadata) + assert metadata["repository"]["url"] == "https://github.com/acme/infra.git" + assert metadata["repository"]["host"] == "github.com" + + +def test_an_scp_style_remote_still_yields_a_host(): + """`git@github.com:acme/infra.git` has no scheme, so urlsplit reads it as a path with no host.""" + metadata = check.build_metadata(_opts(repo_url="git@github.com:acme/infra.git"), redactions=0) + + assert metadata["repository"]["host"] == "github.com" + assert metadata["repository"]["provider"] == "github" + + +def test_a_self_hosted_host_is_unknown_rather_than_guessed(): + """Guessing `github` for git.example.internal would be worse than admitting we cannot tell.""" + metadata = check.build_metadata(_opts(repo_url="https://git.example.internal/acme/infra"), redactions=0) + + assert metadata["repository"]["provider"] == "unknown" + # The raw host is still recorded, which is what makes the honest answer useful. + assert metadata["repository"]["host"] == "git.example.internal" + + +def test_the_declared_repo_path_wins_over_inference(): + opts = _opts(source_dir=".", repo_path="infra/prod") + + code = check.build_metadata(opts, redactions=0)["code"] + + assert code["repo_path"] == "infra/prod" + assert code["repo_path_from"] == "flag" + + +def test_the_repo_path_is_inferred_from_the_enclosing_checkout(tmp_path): + """ + Inference walks up for a `.git` entry rather than shelling out -- this package has no git + dependency, and a `.git` *file* (worktrees, submodules) has to count too. + """ + (tmp_path / ".git").write_text("gitdir: /elsewhere") + nested = tmp_path / "infra" / "prod" + nested.mkdir(parents=True) + + code = check.build_metadata(_opts(source_dir=str(nested)), redactions=0)["code"] + + assert code["repo_path"] == "infra/prod" + assert code["repo_path_from"] == "git_root" + + +def test_the_repository_root_is_the_empty_string_not_a_dot(tmp_path): + """ + `""` means the root and joins correctly; `None` means "we could not tell". Collapsing them would + make a consumer unable to distinguish a root-level project from an unknown one. + """ + (tmp_path / ".git").mkdir() + + code = check.build_metadata(_opts(source_dir=str(tmp_path)), redactions=0)["code"] + + assert code["repo_path"] == "" + assert code["repo_path_from"] == "git_root" + + +def test_an_unlocatable_repository_root_says_so(tmp_path): + code = check.build_metadata(_opts(source_dir=str(tmp_path)), redactions=0)["code"] + + assert code["repo_path"] is None + assert code["repo_path_from"] is None + + +def test_the_oversize_retry_records_that_the_code_was_dropped_for_size(tmp_path, monkeypatch): + """ + The fallback re-packs without the source. A consumer holding only the bundle must be able to tell + that from a deliberate documents-only run, which is the difference between "nothing to fix here" + and "we could not show you the code". + """ + import io + import json as _json + import tarfile + + monkeypatch.setattr(check.archive, "MAX_ARCHIVE_BYTES", 50 * 1024) + source = tmp_path / "src" + source.mkdir() + (source / "main.tf").write_text("") + (source / "vendor.bin").write_bytes(os.urandom(200_000)) + + archive_bytes, _manifest, reason = check.pack_documents( + str(source), + {"masked": True}, + None, + None, + metadata={"schema_version": 1, "code": {}}, + ) + + assert reason + with tarfile.open(fileobj=io.BytesIO(archive_bytes), mode="r:gz") as tar: + metadata = _json.loads(tar.extractfile(check.archive.METADATA_DOCUMENT).read()) + assert metadata["code"]["absent_reason"] == "too_large" + assert metadata["code"]["present"] is False + + +def test_a_declared_repo_path_cannot_escape_the_repository(tmp_path, capsys): + """ + `--repo-path ../..` used to survive `strip("/")` and be recorded verbatim. + + The single use of this field is a consumer joining it to write files back into the repository it + thinks it is patching, so a value that climbs out of the tree is the one shape that must not be + recorded. Refused and left absent rather than recorded wrong -- absent is a state consumers already + handle. + """ + for escaping in ("../..", "/etc", "infra/../../elsewhere"): + code = check.build_metadata(_opts(source_dir=str(tmp_path), repo_path=escaping), redactions=0)["code"] + + assert code["repo_path"] != escaping + assert code["repo_path"] is None or not code["repo_path"].startswith("..") + assert "must be a path inside the repository" in capsys.readouterr().err + + +def test_a_declared_repo_path_is_normalised(tmp_path): + """Leading and trailing slashes, and a redundant `.`, all describe the same location.""" + for declared, expected in (("/infra/prod/", "infra/prod"), ("./infra", "infra"), (".", ""), ("/", "")): + code = check.build_metadata(_opts(source_dir=str(tmp_path), repo_path=declared), redactions=0)["code"] + assert code["repo_path"] == expected, f"{declared!r} -> {code['repo_path']!r}" + assert code["repo_path_from"] == "flag" + + +def test_a_nonexistent_source_dir_fails_rather_than_degrading(tmp_path): + """ + `archive.pack` raises ArchiveError for a missing directory *and* for an oversized archive, and the + degrade path only knew about the second. A typo'd `--source-dir` therefore reported "the tree was + too large", dropped the code and completed the run -- a check that passed having evaluated no + source at all, with the bundle's own metadata stating the wrong reason. + """ + with pytest.raises(check.CheckError, match="--source-dir does not exist"): + check.pack_documents( + str(tmp_path / "no-such-dir"), + {"masked": True}, + None, + None, + metadata={"schema_version": 1, "code": {}}, + ) diff --git a/tests/platform/test_cli_options.py b/tests/platform/test_cli_options.py new file mode 100644 index 00000000..cba60e13 --- /dev/null +++ b/tests/platform/test_cli_options.py @@ -0,0 +1,215 @@ +""" +Tests for `tirith platform check` option handling. + +Everything here is asserted *before* any HTTP call, which is the point: a bad workflow id or a +contradictory pair of URL flags should fail immediately rather than after a run has been created. +""" + +import json + +import pytest + +from tirith.platform import cli +from tirith.status import ExitStatus + +PLAN = {"format_version": "1.2", "resource_changes": []} + +# The minimum run_check result cli.main will accept without reaching for a missing key. +PASSED = {"verdict": "passed", "counts": {}, "policies": {}} + + +@pytest.fixture +def no_network(monkeypatch): + """Make any attempt to reach the platform an outright test failure.""" + + def explode(*a, **kw): + raise AssertionError("run_check was called; the option check should have failed first") + + monkeypatch.setattr(cli, "run_check", explode) + + +def base_args(tmp_path, *extra): + plan = tmp_path / "plan.json" + plan.write_text(json.dumps(PLAN)) + return ["platform", "check", "--input-path", str(plan), *extra] + + +def env(monkeypatch, **values): + for key in ("SG_API_TOKEN", "SG_ORG", "SG_BASE_URL", "SG_DASHBOARD_URL", "SG_REGION"): + monkeypatch.delenv(key, raising=False) + for key, value in values.items(): + monkeypatch.setenv(key, value) + + +class TestWorkflowIdValidation: + @pytest.mark.parametrize("workflow_id", ["live/prod/vpc", "has.dots", "a" * 101, "spaces here", ""]) + def test_a_bad_slug_is_refused_before_any_request(self, workflow_id, tmp_path, monkeypatch, no_network, capsys): + """ + The value is interpolated into every API path and the platform's own field is a slug, so a + '/' produces a malformed URL rather than a clear error. + """ + env(monkeypatch, SG_API_TOKEN="sgo_x", SG_ORG="acme") + + status = cli.main(base_args(tmp_path, "--workflow-id", workflow_id)) + + assert status == ExitStatus.ERROR + assert "not a valid slug" in capsys.readouterr().err + + def test_the_error_suggests_a_usable_slug(self, tmp_path, monkeypatch, no_network, capsys): + env(monkeypatch, SG_API_TOKEN="sgo_x", SG_ORG="acme") + + cli.main(base_args(tmp_path, "--workflow-id", "live/prod/vpc")) + + assert "live-prod-vpc" in capsys.readouterr().err + + @pytest.mark.parametrize("workflow_id", ["github-com-acme-infra-plan", "a_b-C9", "x"]) + def test_valid_slugs_pass(self, workflow_id, tmp_path, monkeypatch, capsys): + env(monkeypatch, SG_API_TOKEN="sgo_x", SG_ORG="acme") + seen = {} + + def capture(opts): + seen["workflow_id"] = opts.workflow_id + return PASSED + + monkeypatch.setattr(cli, "run_check", capture) + cli.main(base_args(tmp_path, "--workflow-id", workflow_id)) + + assert seen["workflow_id"] == workflow_id + + +class TestRegionResolution: + def resolved(self, tmp_path, monkeypatch, *extra): + seen = {} + + def capture(opts): + seen["api_url"] = opts.api_url + seen["dashboard_url"] = opts.dashboard_url + return PASSED + + monkeypatch.setattr(cli, "run_check", capture) + status = cli.main(base_args(tmp_path, "--workflow-id", "wf", *extra)) + return status, seen + + def test_region_us_sets_both_urls(self, tmp_path, monkeypatch): + env(monkeypatch, SG_API_TOKEN="sgo_x", SG_ORG="acme") + + _status, seen = self.resolved(tmp_path, monkeypatch, "--region", "us") + + assert seen["api_url"] == "https://api.us.stackguardian.io/api/v1" + assert seen["dashboard_url"] == "https://us.stackguardian.io" + + def test_defaults_to_eu(self, tmp_path, monkeypatch): + env(monkeypatch, SG_API_TOKEN="sgo_x", SG_ORG="acme") + + _status, seen = self.resolved(tmp_path, monkeypatch) + + assert seen["api_url"] == "https://api.app.stackguardian.io/api/v1" + assert seen["dashboard_url"] == "https://app.stackguardian.io" + + def test_region_with_an_explicit_url_fails_before_any_request(self, tmp_path, monkeypatch, no_network, capsys): + env(monkeypatch, SG_API_TOKEN="sgo_x", SG_ORG="acme") + + status = cli.main( + base_args(tmp_path, "--workflow-id", "wf", "--region", "us", "--api-url", "https://x.example") + ) + + assert status == ExitStatus.ERROR + assert "cannot be combined" in capsys.readouterr().err + + def test_an_unknown_region_is_rejected_by_the_parser(self, tmp_path, monkeypatch): + env(monkeypatch, SG_API_TOKEN="sgo_x", SG_ORG="acme") + + with pytest.raises(SystemExit): + cli.main(base_args(tmp_path, "--workflow-id", "wf", "--region", "uss")) + + def test_a_base_url_without_the_api_path_still_works(self, tmp_path, monkeypatch): + """A SG_BASE_URL exported for sg-cli omits /api/v1 and used to 404 here.""" + env(monkeypatch, SG_API_TOKEN="sgo_x", SG_ORG="acme", SG_BASE_URL="https://api.us.stackguardian.io") + + _status, seen = self.resolved(tmp_path, monkeypatch) + + assert seen["api_url"] == "https://api.us.stackguardian.io/api/v1" + + def test_setting_only_the_api_url_still_gets_correct_run_links(self, tmp_path, monkeypatch): + """The original footgun: run links pointed at the EU dashboard for a US org.""" + env(monkeypatch, SG_API_TOKEN="sgo_x", SG_ORG="acme") + + _status, seen = self.resolved(tmp_path, monkeypatch, "--api-url", "https://api.us.stackguardian.io") + + assert seen["dashboard_url"] == "https://us.stackguardian.io" + + +class TestDocumentSelection: + def test_a_plan_is_discovered_when_nothing_is_named(self, tmp_path, monkeypatch): + (tmp_path / "plan.json").write_text(json.dumps(PLAN)) + env(monkeypatch, SG_API_TOKEN="sgo_x", SG_ORG="acme") + seen = {} + monkeypatch.setattr(cli, "run_check", lambda opts: seen.update(input_path=opts.input_path) or PASSED) + + cli.main(["platform", "check", "--workflow-id", "wf", "--source-dir", str(tmp_path)]) + + assert seen["input_path"].endswith("plan.json") + + def test_nothing_to_evaluate_is_an_error(self, tmp_path, monkeypatch, no_network, capsys): + env(monkeypatch, SG_API_TOKEN="sgo_x", SG_ORG="acme") + + status = cli.main(["platform", "check", "--workflow-id", "wf", "--source-dir", str(tmp_path)]) + + assert status == ExitStatus.ERROR + assert "No plan document found" in capsys.readouterr().err + + def test_plan_file_and_input_path_cannot_be_combined(self, tmp_path, monkeypatch, no_network, capsys): + env(monkeypatch, SG_API_TOKEN="sgo_x", SG_ORG="acme") + + status = cli.main(base_args(tmp_path, "--workflow-id", "wf", "--plan-file", str(tmp_path / "tfplan"))) + + assert status == ExitStatus.ERROR + assert "cannot be combined" in capsys.readouterr().err + + def test_an_explicit_input_path_skips_discovery(self, tmp_path, monkeypatch): + """Two candidates would be ambiguous for discovery, but naming one is unambiguous.""" + (tmp_path / "plan.json").write_text(json.dumps(PLAN)) + (tmp_path / "tfplan.json").write_text(json.dumps(PLAN)) + env(monkeypatch, SG_API_TOKEN="sgo_x", SG_ORG="acme") + seen = {} + monkeypatch.setattr(cli, "run_check", lambda opts: seen.update(input_path=opts.input_path) or PASSED) + + status = cli.main( + [ + "platform", + "check", + "--workflow-id", + "wf", + "--source-dir", + str(tmp_path), + "--input-path", + str(tmp_path / "tfplan.json"), + ] + ) + + assert status != ExitStatus.ERROR + assert seen["input_path"].endswith("tfplan.json") + + +class TestCredentials: + def test_credentials_come_from_the_environment(self, tmp_path, monkeypatch): + """ + The one-liner needs this: GitHub exposes neither secrets nor vars as env automatically, so + an `env:` block is the only no-`with:` route. + """ + env(monkeypatch, SG_API_TOKEN="sgo_fromenv", SG_ORG="acme-from-env") + seen = {} + monkeypatch.setattr(cli, "run_check", lambda opts: seen.update(api_key=opts.api_key, org=opts.org) or PASSED) + + cli.main(base_args(tmp_path, "--workflow-id", "wf")) + + assert seen == {"api_key": "sgo_fromenv", "org": "acme-from-env"} + + def test_missing_credentials_name_both(self, tmp_path, monkeypatch, no_network, capsys): + env(monkeypatch) + + status = cli.main(base_args(tmp_path, "--workflow-id", "wf")) + + assert status == ExitStatus.ERROR + err = capsys.readouterr().err + assert "--api-key" in err and "--org" in err diff --git a/tests/platform/test_client.py b/tests/platform/test_client.py new file mode 100644 index 00000000..0b44ab2c --- /dev/null +++ b/tests/platform/test_client.py @@ -0,0 +1,666 @@ +""" +Tests for the StackGuardian client. + +The polling contract is the part worth pinning: a run that rests in a state the poller does not +recognise as terminal spins until the timeout and is then reported as a tool failure -- turning a +completed evaluation into what looks like an outage. +""" + +import json + +import pytest + +from tirith.platform import client +from tirith.platform.client import SGClient, SGError, _extract_signed_url + +# --- terminal statuses ------------------------------------------------------------------------- + + +def test_approval_required_is_terminal(): + """ + A regression test. APPROVAL_REQUIRED is a resting state -- reached when a policy's onFail is + APPROVAL_REQUIRED -- and nothing further happens without a human. Treating it as transient + made the poller spin to its timeout and report a tool failure for a finished evaluation. + """ + assert "APPROVAL_REQUIRED" in client.TERMINAL_STATUSES + + +@pytest.mark.parametrize("status", ["COMPLETED", "ERRORED", "CANCELLED", "APPROVAL_REQUIRED"]) +def test_terminal_statuses_stop_the_poll(status): + assert status in client.TERMINAL_STATUSES + + +@pytest.mark.parametrize("status", ["QUEUED", "PENDING", "RUNNING"]) +def test_transient_statuses_keep_polling(status): + """A run can sit in QUEUED behind the per-workflow concurrency gate for a long while.""" + assert status not in client.TERMINAL_STATUSES + + +def test_wait_for_run_returns_on_a_terminal_status(monkeypatch): + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + statuses = iter([{"LatestStatus": "QUEUED"}, {"LatestStatus": "RUNNING"}, {"LatestStatus": "COMPLETED"}]) + monkeypatch.setattr(sg, "get_run", lambda *a, **k: next(statuses)) + monkeypatch.setattr(client.time, "sleep", lambda _s: None) + + status, _run = sg.wait_for_run("default", "wf", "run", timeout=30) + + assert status == "COMPLETED" + + +def test_wait_for_run_reports_each_status_change(monkeypatch): + """Without this a run queued behind another looks identical to a hung one.""" + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + statuses = iter([{"LatestStatus": "QUEUED"}, {"LatestStatus": "QUEUED"}, {"LatestStatus": "COMPLETED"}]) + monkeypatch.setattr(sg, "get_run", lambda *a, **k: next(statuses)) + monkeypatch.setattr(client.time, "sleep", lambda _s: None) + seen = [] + + sg.wait_for_run("default", "wf", "run", timeout=30, on_poll=seen.append) + + assert seen == ["QUEUED", "COMPLETED"], "only changes are reported, not every poll" + + +def test_wait_for_run_timeout_is_an_error_never_a_pass(monkeypatch): + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + monkeypatch.setattr(sg, "get_run", lambda *a, **k: {"LatestStatus": "RUNNING"}) + monkeypatch.setattr(client.time, "sleep", lambda _s: None) + + with pytest.raises(SGError): + sg.wait_for_run("default", "wf", "run", timeout=-1) + + +# --- signed URL extraction --------------------------------------------------------------------- + + +def test_extract_signed_url_accepts_a_bare_string_in_msg(): + """What tfstate_upload_url actually returns.""" + assert _extract_signed_url({"msg": "https://s3.example/put"}) == "https://s3.example/put" + + +def test_extract_signed_url_accepts_a_nested_object(): + assert _extract_signed_url({"data": {"signedUrl": "https://s3.example/put"}}) == "https://s3.example/put" + + +def test_extract_signed_url_returns_none_when_absent(): + assert _extract_signed_url({"msg": "some error text"}) is None + + +# --- archive upload ---------------------------------------------------------------------------- + + +def _fake_put(recorder): + """Stand in for the presigned PUT, recording what was sent.""" + + def fake_urlopen(request, timeout=None): + recorder["content_type"] = request.get_header("Content-type") + recorder["body"] = request.data + + class _R: + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + return _R() + + return fake_urlopen + + +def test_upload_archive_tolerates_a_response_with_no_storage_key(monkeypatch): + """ + The key used to be mandatory, because the caller passed it back as a run field and an api that did + not return it produced a run pointing at nothing. Nothing passes it anywhere now -- the step finds + the bundle by name in the artifacts directory -- so an api that omits it must not fail the upload. + + This is what lets the feature ship against an unmodified api. + """ + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + monkeypatch.setattr(sg, "_request", lambda *a, **k: (200, {"msg": "https://s3.example/put"})) + monkeypatch.setattr(client.urllib.request, "urlopen", _fake_put({})) + + key = sg.upload_file("default", "wf", "a.tar.gz", None, b"x") + + assert "a.tar.gz" in key + + +def _upload_response(): + """What file_upload_url returns: the URL as a bare string in msg, the key alongside in data.""" + return (200, {"msg": "https://s3.example/put", "data": {"key": "orgs/acme/wfs/K/artifacts/abc1234/a.tar.gz"}}) + + +def test_upload_archive_returns_the_key_from_the_response(monkeypatch): + """ + Never rebuilt client-side: the layout depends on ArtifactsUnderKSUID, ResourceKSUID and + OriginalArtifactPath, so a guess is wrong for exactly the customers hardest to debug. + """ + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + monkeypatch.setattr(sg, "_request", lambda *a, **k: _upload_response()) + uploaded = {} + + def fake_urlopen(request, timeout=None): + uploaded["content_type"] = request.get_header("Content-type") + uploaded["body"] = request.data + + class _R: + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + return _R() + + monkeypatch.setattr(client.urllib.request, "urlopen", fake_urlopen) + + key = sg.upload_file("default", "wf", "a.tar.gz", "abc1234", b"tarbytes") + + assert key == "orgs/acme/wfs/K/artifacts/abc1234/a.tar.gz" + assert uploaded["body"] == b"tarbytes" + # application/json even though the body is gzip: the endpoint signs application/json whatever + # the filename, and S3 validates the signature against the header the client sends. Asking for + # application/gzip would need an api change, and sending it unasked earns SignatureDoesNotMatch. + assert uploaded["content_type"] == "application/json" + + +def test_upload_archive_uses_the_shared_artifact_endpoint(monkeypatch): + """ + Not a bespoke endpoint. The bundle has to land in the workflow's own artifact prefix, because + that prefix is what the runner syncs down into the step -- so it uploads through the same route + every other artifact uses. + + And it must ask for nothing the endpoint does not already offer: no contentType parameter, since + signing anything other than application/json would need an api change this feature avoids. + """ + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + seen = {} + + def fake_request(method, path, *a, **k): + seen["method"] = method + seen["path"] = path + return _upload_response() + + monkeypatch.setattr(sg, "_request", fake_request) + monkeypatch.setattr(client.urllib.request, "urlopen", _ok_urlopen()) + + sg.upload_file("default", "wf", "a.tar.gz", "abc1234", b"tarbytes") + + assert seen["method"] == "GET" + assert "/file_upload_url/" in seen["path"] + assert "configuration_upload_url" not in seen["path"] + assert "contentType" not in seen["path"], "asking for a signed content type needs an api change" + assert "filename=a.tar.gz" in seen["path"] + + +def _ok_urlopen(): + def fake_urlopen(request, timeout=None): + class _R: + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + return _R() + + return fake_urlopen + + +# --- run creation ------------------------------------------------------------------------------ + + +def test_create_run_sends_no_step_config(monkeypatch): + """ + core ignores WfStepsConfig for TERRAFORM workflows and synthesises the steps from the stored + TerraformConfig plus this TerraformAction. Sending one would be dead weight that reads as if + it were doing something. + """ + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + captured = {} + + def fake_request(method, path, body=None, **kwargs): + captured["body"] = body + return 200, {"data": {"ResourceName": "wfrun-1"}} + + monkeypatch.setattr(sg, "_request", fake_request) + + run_id, _data = sg.create_run("default", "wf", {"type": "tirith"}) + + assert run_id == "wfrun-1" + assert "WfStepsConfig" not in captured["body"] + # `plan` is a dummy: the policy step is spliced in ahead of the plan step and exits 12, so the + # plan never runs. `plan` is simply the action whose synthesis splices pre-plan steps in. + assert captured["body"]["TerraformAction"] == {"action": "plan"} + # No archive field of any kind. The bundle reaches the step through the workflow's artifact + # directory, which is what lets this run against an unmodified api -- so a field appearing here + # again would mean the api dependency had come back. + assert "terraformProjectZip" not in captured["body"] + assert "CodeZipWfArtifactPath" not in captured["body"] + assert "ContextTags" not in captured["body"] + + +def test_create_run_does_not_depend_on_the_platform_echoing_an_archive_field(monkeypatch): + """ + There used to be a guard here: the run body carried `terraformProjectZip`, an api that did not + declare it dropped it silently during validation, and the run then evaluated a VCS checkout instead + of the uploaded code. The guard asserted the field back out of RuntimeParameters. + + It is gone because the cause is gone -- nothing is sent for the platform to drop. A run whose + RuntimeParameters mention no archive at all is now completely normal, and must not fail. + """ + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + monkeypatch.setattr( + sg, + "_request", + lambda *a, **k: (200, {"data": {"ResourceName": "wfrun-1", "RuntimeParameters": {"vcsConfig": {}}}}), + ) + + run_id, _data = sg.create_run("default", "wf", {"type": "tirith"}) + + assert run_id == "wfrun-1" + + +def test_create_run_accepts_a_response_that_carries_no_runtime_parameters(monkeypatch): + """ + A response shape without RuntimeParameters is not evidence the field was dropped, and failing on + it would break the client against a platform that is behaving correctly. + """ + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + monkeypatch.setattr(sg, "_request", lambda *a, **k: (200, {"data": {"ResourceName": "wfrun-1"}})) + + run_id, _data = sg.create_run("default", "wf", {"type": "tirith"}) + + assert run_id == "wfrun-1" + + +def test_create_run_passes_when_the_platform_stored_the_archive_reference(monkeypatch): + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + monkeypatch.setattr( + sg, + "_request", + lambda *a, **k: ( + 200, + {"data": {"ResourceName": "wfrun-1", "RuntimeParameters": {"terraformProjectZip": "orgs/acme/a.tar.gz"}}}, + ), + ) + + run_id, _data = sg.create_run("default", "wf", {"type": "tirith"}) + + assert run_id == "wfrun-1" + + +def test_ensure_workflow_creates_a_terraform_workflow(monkeypatch): + """ + TERRAFORM rather than CUSTOM: it is what makes core synthesise the steps from TerraformConfig, + and what makes the run render as a real terraform run in the dashboard. + """ + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + captured = {} + + def fake_request(method, path, body=None, **kwargs): + captured["body"] = body + return 201, {} + + monkeypatch.setattr(sg, "_request", fake_request) + + sg.ensure_workflow("default", "wf", "desc", {"terraformVersion": "1.5.7"}) + + assert captured["body"]["WfType"] == "TERRAFORM" + assert captured["body"]["TerraformConfig"] == {"terraformVersion": "1.5.7"} + assert captured["body"]["Id"] == captured["body"]["ResourceName"] == "wf" + + +def test_conflict_on_create_is_success(monkeypatch): + """Re-running the action against an existing workflow must not be an error.""" + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + monkeypatch.setattr(sg, "_request", lambda *a, **k: (409, {"msg": "already exists"})) + + assert sg.ensure_workflow("default", "wf", "d", {}) == 409 + assert sg.ensure_workflow_group("default") == 409 + + +# --- auth -------------------------------------------------------------------------------------- + + +def test_auth_header_uses_the_apikey_scheme(monkeypatch): + """Matches sg-cli: `Authorization: apikey `, not Bearer.""" + sg = SGClient("https://api.example/api/v1", "acme", "sgo_secret") + captured = {} + + def fake_urlopen(request, timeout=None): + captured["auth"] = request.get_header("Authorization") + + class _R: + status = 200 + + def read(self): + return json.dumps({"msg": "ok"}).encode() + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + return _R() + + monkeypatch.setattr(client.urllib.request, "urlopen", fake_urlopen) + + sg._request("GET", "/wfgrps/") + + assert captured["auth"] == "apikey sgo_secret" + + +# --- run facts and cleanup ---------------------------------------------------------------------- + + +def test_policy_results_follow_the_snake_case_signed_url(monkeypatch): + """ + The facts endpoint returns `signed_url`; this used to read only `signedUrl` and so always + returned {}. It went unnoticed for as long as the results artifact was covering for it. + """ + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + monkeypatch.setattr(sg, "_request", lambda *a, **k: (200, {"msg": {"signed_url": "https://s3.example/facts"}})) + + class _R: + def read(self): + return json.dumps({"PolicyEvalResults": {"p": [{"result": "PASS"}]}}).encode() + + def info(self): + return {} + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + monkeypatch.setattr(client.urllib.request, "urlopen", lambda *a, **k: _R()) + + assert sg.get_policy_results("default", "wf", "run-1") == {"p": [{"result": "PASS"}]} + + +def test_policy_results_accept_an_inline_payload(monkeypatch): + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + monkeypatch.setattr( + sg, "_request", lambda *a, **k: (200, {"msg": {"PolicyEvalResults": {"p": [{"result": "FAIL"}]}}}) + ) + + assert sg.get_policy_results("default", "wf", "run-1") == {"p": [{"result": "FAIL"}]} + + +def test_missing_results_artifact_is_none_not_empty(monkeypatch): + """ + The caller distinguishes "no such artifact, the facts are authoritative" from "the artifact + exists and no policies matched". Collapsing both to {} would hide a real no-policies verdict. + """ + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + monkeypatch.setattr(sg, "_request", lambda *a, **k: (404, {"msg": "not found"})) + + assert sg.get_results_artifact("default", "wf", "run-1/tirith-results.json") is None + + +@pytest.mark.parametrize("status", [200, 204, 404]) +def test_delete_artifact_treats_absence_as_success(monkeypatch, status): + """404 means someone already removed it, which is the state we wanted.""" + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + monkeypatch.setattr(sg, "_request", lambda *a, **k: (status, {})) + + assert sg.delete_artifact("default", "wf", "__sg.abc1234-default.tar.gz") is True + + +def test_delete_artifact_reports_failure_rather_than_raising(monkeypatch): + """Cleanup runs after the verdict is known, so a failure must not change the outcome.""" + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + monkeypatch.setattr(sg, "_request", lambda *a, **k: (403, {"msg": "denied"})) + + assert sg.delete_artifact("default", "wf", "__sg.abc1234-default.tar.gz") is False + + +def test_delete_artifact_targets_a_single_path_segment(monkeypatch): + """ + A nested name is swallowed by the greedy converter in the authorizer and matches + `DELETE .../wfgrps//` -- the workflow-group delete -- so it would be checked against + entirely the wrong permission. + """ + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + seen = {} + monkeypatch.setattr(sg, "_request", lambda m, p, *a, **k: (seen.update(method=m, path=p), (200, {}))[1]) + + sg.delete_artifact("default", "wf", "__sg.abc1234-default.tar.gz") + + assert seen["method"] == "DELETE" + tail = seen["path"].split("/artifacts/", 1)[1].rstrip("/") + assert "/" not in tail, f"artifact name must be one segment, got {tail!r}" + + +@pytest.mark.parametrize("folder", [None, ""]) +def test_upload_archive_omits_an_unset_folder(monkeypatch, folder): + """ + urlencode stringifies None to the literal "None", and the endpoint treats any non-empty value + as a subfolder -- so passing it unconditionally created a real `None/` directory in S3 and left + the archive at a nested key the post-run delete could not address. Caught in QA. + """ + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + seen = {} + monkeypatch.setattr(sg, "_request", lambda m, p, *a, **k: (seen.update(path=p), _upload_response())[1]) + monkeypatch.setattr(client.urllib.request, "urlopen", _ok_urlopen()) + + sg.upload_file("default", "wf", "__sg.abc1234-default.tar.gz", folder, b"tarbytes") + + assert "folder=" not in seen["path"], seen["path"] + assert "None" not in seen["path"], seen["path"] + + +def test_upload_archive_sends_a_folder_when_one_is_given(monkeypatch): + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + seen = {} + monkeypatch.setattr(sg, "_request", lambda m, p, *a, **k: (seen.update(path=p), _upload_response())[1]) + monkeypatch.setattr(client.urllib.request, "urlopen", _ok_urlopen()) + + sg.upload_file("default", "wf", "a.tar.gz", "abc1234", b"tarbytes") + + assert "folder=abc1234" in seen["path"] + + +# --- publishing the state document --------------------------------------------------------------- + + +def test_upload_file_honours_a_json_content_type(monkeypatch): + """ + The state document is JSON, not a gzip. S3 signs the content type into the URL, so sending the + archive's type with a JSON body is a signature mismatch. + """ + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + captured = {} + + def fake_request(method, path, **kwargs): + captured["path"] = path + return _upload_response() + + monkeypatch.setattr(sg, "_request", fake_request) + uploaded = {} + + def fake_urlopen(request, timeout=None): + uploaded["content_type"] = request.get_header("Content-type") + uploaded["body"] = request.data + + class _R: + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + return _R() + + monkeypatch.setattr(client.urllib.request, "urlopen", fake_urlopen) + + sg.upload_file("default", "wf", "tfstate.json", None, b'{"version": 4}', content_type="application/json") + + assert uploaded["content_type"] == "application/json" + assert uploaded["body"] == b'{"version": 4}' + # The endpoint already signs application/json, so nothing has to be asked for. + assert "contentType" not in captured["path"] + + +def test_manages_terraform_state_reads_the_workflow_config(monkeypatch): + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + + monkeypatch.setattr( + sg, "_request", lambda *a, **k: (200, {"msg": {"TerraformConfig": {"managedTerraformState": True}}}) + ) + assert sg.manages_terraform_state("default", "wf") is True + + monkeypatch.setattr( + sg, "_request", lambda *a, **k: (200, {"msg": {"TerraformConfig": {"managedTerraformState": False}}}) + ) + assert sg.manages_terraform_state("default", "wf") is False + + +@pytest.mark.parametrize( + "response", + [ + (404, {"msg": "not found"}), + (500, {"msg": "boom"}), + (200, {"msg": "a string, not a dict"}), + (200, {}), + ], +) +def test_an_unreadable_workflow_is_treated_as_managing_its_own_state(monkeypatch, response): + """ + Fails safe. Not being able to tell whether `artifacts/tfstate.json` is live terraform state is + not a reason to overwrite it with a masked document. + """ + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + monkeypatch.setattr(sg, "_request", lambda *a, **k: response) + + assert sg.manages_terraform_state("default", "wf") is True + + +def test_an_absent_facts_document_is_not_a_read_failure(monkeypatch): + """ + 404 means the run produced no facts document, which is a legitimate empty result. Treating it + as unreadable would turn healthy runs red -- the opposite of the mistake the raise exists to fix. + """ + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + monkeypatch.setattr(sg, "_request", lambda *a, **k: (404, {"msg": "not found"})) + + assert sg.get_run_facts("default", "wf", "run-1") == {} + + +def test_an_unreadable_facts_document_raises_rather_than_reading_as_empty(monkeypatch): + """ + A 403 or a 500 means we could not read the verdict, not that there was none. Returning {} made + that indistinguishable from "no policies in scope", so a run whose policies had failed reported + a clean scope and exited 0. + """ + for status in (403, 500, 502): + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + monkeypatch.setattr(sg, "_request", lambda *a, **k: (status, {"msg": "nope"})) + + with pytest.raises(SGError, match="Could not read the run facts"): + sg.get_run_facts("default", "wf", "run-1") + + +def test_create_run_sends_the_bundle_name_in_terraform_config(monkeypatch): + """ + The per-run channel, and the only one that works for a TERRAFORM workflow. + + core ignores a run's `WfStepsConfig` for TERRAFORM and synthesises the steps from TerraformConfig + instead, so a step entry has to travel inside `TerraformConfig.prePlanWfStepsConfig` to reach the + run at all. Verified against QA: the same entry sent as top-level WfStepsConfig was silently + discarded and the step kept the workflow's stored path. + """ + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + captured = {} + + def fake_request(method, path, body=None, **kwargs): + captured["body"] = body + return 200, {"data": {"ResourceName": "wfrun-1"}} + + monkeypatch.setattr(sg, "_request", fake_request) + step = { + "name": "tirith-iac-governance", + "wfStepInputData": {"data": {"bundlePath": "tirith-bundle-a1b2c3d-plan.tar.gz"}}, + } + + sg.create_run("default", "wf", {"type": "tirith"}, pre_plan_steps=[step]) + + sent = captured["body"]["TerraformConfig"]["prePlanWfStepsConfig"] + assert sent[0]["wfStepInputData"]["data"]["bundlePath"] == "tirith-bundle-a1b2c3d-plan.tar.gz" + # Only prePlanWfStepsConfig: core's merge is shallow, so sending terraformVersion or + # managedTerraformState here would override what the workflow stores rather than inherit it. + assert set(captured["body"]["TerraformConfig"]) == {"prePlanWfStepsConfig"} + + +def test_create_run_without_steps_sends_no_terraform_config(monkeypatch): + """A caller that names no bundle must not blank the workflow's stored configuration.""" + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + captured = {} + monkeypatch.setattr( + sg, + "_request", + lambda m, p, body=None, **k: (captured.setdefault("body", body), (200, {"data": {"ResourceName": "r"}}))[1], + ) + + sg.create_run("default", "wf", {"type": "tirith"}) + + assert "TerraformConfig" not in captured["body"] + + +def test_every_run_suppresses_the_vcs_checkout(monkeypatch): + """ + The run must send an *empty* VCSConfig, and must send it even when nothing else is set. + + core resolves the run's config as `data.get("VCSConfig", wfDetails.get("VCSConfig", {}))`, so a + present empty value beats the workflow's and an omitted key inherits it. Inheriting is what broke + private repositories: the runner cloned with no credentials and the run ERRORED before the step + ran. Asserting `== {}` rather than truthiness is the point -- `None` would also read as "no + checkout" here but flows into core's config-policy payload as a null. + """ + for kwargs in ({}, {"pre_plan_steps": [{"name": "tirith-iac-governance"}]}): + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + captured = {} + + def fake_request(method, path, body=None, **kw): + captured["body"] = body + return 200, {"data": {"ResourceName": "r"}} + + monkeypatch.setattr(sg, "_request", fake_request) + sg.create_run("default", "wf", {"type": "tirith"}, **kwargs) + + assert "VCSConfig" in captured["body"], "an omitted key inherits the workflow's repo" + assert captured["body"]["VCSConfig"] == {} + + +def test_the_workflow_still_records_its_repository(monkeypatch): + """ + Suppressing the checkout per run must not cost the workflow its repo link -- that is the whole + reason the config is set on creation, and the two live at different levels for that reason. + """ + sg = SGClient("https://api.example/api/v1", "acme", "sgo_x") + captured = {} + + def fake_request(method, path, body=None, **kw): + captured["body"] = body + return 201, {} + + monkeypatch.setattr(sg, "_request", fake_request) + vcs = SGClient.vcs_config("https://github.com/acme/repo", "main") + sg.ensure_workflow("default", "wf", "d", {"terraformVersion": "1.5.7"}, vcs_config=vcs) + + source = captured["body"]["VCSConfig"]["iacVCSConfig"]["customSource"] + assert source["config"]["repo"] == "https://github.com/acme/repo" + assert source["sourceConfigDestKind"] == "GIT_OTHER" + # api rejects iacVCSConfig without it, so it is always present at this level -- which is exactly + # why the run has to send an empty config rather than a trimmed one. + assert captured["body"]["VCSConfig"]["iacVCSConfig"]["useMarketplaceTemplate"] is False diff --git a/tests/platform/test_discover.py b/tests/platform/test_discover.py new file mode 100644 index 00000000..f780210f --- /dev/null +++ b/tests/platform/test_discover.py @@ -0,0 +1,229 @@ +""" +Tests for convention-based document discovery and `terraform show -json`. + +The property worth protecting hardest is in `test_the_plan_never_reaches_github_output`: calling the +CI wrapper instead of the real binary copies the entire unmasked plan into $GITHUB_OUTPUT, a file +every later step in the job can read. +""" + +import json +import os +import stat + +import pytest + +from tirith.platform import discover +from tirith.platform.discover import DiscoveryError + +PLAN = {"format_version": "1.2", "resource_changes": []} + + +def write(path, content): + path.write_text(content if isinstance(content, str) else json.dumps(content)) + return path + + +def fake_binary(directory, name, script): + """Drop an executable shell script on disk to stand in for terraform.""" + directory.mkdir(parents=True, exist_ok=True) + path = directory / name + path.write_text(script) + path.chmod(path.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + return path + + +class TestDiscoverInput: + def test_finds_plan_json(self, tmp_path): + write(tmp_path / "plan.json", PLAN) + assert discover.discover_input(str(tmp_path)) == os.path.join(str(tmp_path), "plan.json") + + def test_finds_tfplan_json(self, tmp_path): + write(tmp_path / "tfplan.json", PLAN) + assert discover.discover_input(str(tmp_path)) == os.path.join(str(tmp_path), "tfplan.json") + + def test_two_candidates_is_an_error(self, tmp_path): + """ + Not "first one wins": silently evaluating the wrong document reports a verdict about + infrastructure the caller did not ask about, and it looks like a pass. + """ + write(tmp_path / "plan.json", PLAN) + write(tmp_path / "tfplan.json", PLAN) + + with pytest.raises(DiscoveryError) as excinfo: + discover.discover_input(str(tmp_path)) + + assert "plan.json" in str(excinfo.value) + assert "tfplan.json" in str(excinfo.value) + assert "--input-path" in str(excinfo.value) + + def test_no_candidate_names_every_way_out(self, tmp_path): + with pytest.raises(DiscoveryError) as excinfo: + discover.discover_input(str(tmp_path)) + + message = str(excinfo.value) + assert "plan.json" in message and "tfplan.json" in message + assert "--plan-file" in message + assert "--input-path" in message + + def test_is_not_recursive(self, tmp_path): + """A plan in a subdirectory belongs to a different unit; picking it up would be wrong.""" + (tmp_path / "modules").mkdir() + write(tmp_path / "modules" / "plan.json", PLAN) + + with pytest.raises(DiscoveryError): + discover.discover_input(str(tmp_path)) + + def test_ignores_other_json_in_the_directory(self, tmp_path): + """Two fixed names, not a glob -- a glob would sweep up infracost.json or package.json.""" + write(tmp_path / "infracost.json", {"projects": []}) + write(tmp_path / "package.json", {}) + + with pytest.raises(DiscoveryError): + discover.discover_input(str(tmp_path)) + + def test_a_directory_named_plan_json_is_not_a_document(self, tmp_path): + (tmp_path / "plan.json").mkdir() + + with pytest.raises(DiscoveryError): + discover.discover_input(str(tmp_path)) + + +class TestResolveBinary: + def test_prefers_terraform_bin_over_terraform(self, tmp_path, monkeypatch): + """ + setup-terraform installs a JS wrapper as `terraform` and moves the real binary to + `terraform-bin`. Calling the wrapper leaks the plan into $GITHUB_OUTPUT. + """ + bindir = tmp_path / "bin" + fake_binary(bindir, "terraform", "#!/bin/sh\nexit 0\n") + fake_binary(bindir, "terraform-bin", "#!/bin/sh\nexit 0\n") + monkeypatch.setenv("PATH", str(bindir)) + + assert os.path.basename(discover._resolve_binary()) == "terraform-bin" + + def test_uses_terraform_cli_path_when_set(self, tmp_path, monkeypatch): + bindir = tmp_path / "toolcache" + fake_binary(bindir, "terraform-bin", "#!/bin/sh\nexit 0\n") + otherdir = tmp_path / "bin" + fake_binary(otherdir, "terraform", "#!/bin/sh\nexit 0\n") + monkeypatch.setenv("PATH", str(otherdir)) + monkeypatch.setenv("TERRAFORM_CLI_PATH", str(bindir)) + + assert discover._resolve_binary() == str(bindir / "terraform-bin") + + def test_falls_back_to_tofu(self, tmp_path, monkeypatch): + bindir = tmp_path / "bin" + fake_binary(bindir, "tofu", "#!/bin/sh\nexit 0\n") + monkeypatch.setenv("PATH", str(bindir)) + monkeypatch.delenv("TERRAFORM_CLI_PATH", raising=False) + monkeypatch.delenv("TOFU_CLI_PATH", raising=False) + + assert os.path.basename(discover._resolve_binary()) == "tofu" + + def test_an_explicit_binary_wins(self, tmp_path, monkeypatch): + bindir = tmp_path / "bin" + fake_binary(bindir, "terraform-bin", "#!/bin/sh\nexit 0\n") + monkeypatch.setenv("PATH", str(bindir)) + + assert discover._resolve_binary("/opt/custom/tofu") == "/opt/custom/tofu" + + def test_nothing_found_says_what_to_do(self, tmp_path, monkeypatch): + monkeypatch.setenv("PATH", str(tmp_path / "empty")) + monkeypatch.delenv("TERRAFORM_CLI_PATH", raising=False) + monkeypatch.delenv("TOFU_CLI_PATH", raising=False) + + with pytest.raises(DiscoveryError, match="--terraform-bin"): + discover._resolve_binary() + + +class TestTerraformShowJson: + def test_returns_the_parsed_plan(self, tmp_path, monkeypatch): + bindir = tmp_path / "bin" + fake_binary(bindir, "terraform-bin", f"#!/bin/sh\necho '{json.dumps(PLAN)}'\n") + monkeypatch.setenv("PATH", str(bindir)) + plan_file = tmp_path / "tfplan" + plan_file.write_bytes(b"binary") + + assert discover.terraform_show_json(str(plan_file)) == PLAN + + def test_the_plan_never_reaches_github_output(self, tmp_path, monkeypatch): + """ + The regression that motivates the whole resolution order. `terraform-bin` is the real + binary; the `terraform` beside it is the wrapper, which would append the plan to + $GITHUB_OUTPUT. That file must still be empty afterwards. + """ + bindir = tmp_path / "bin" + github_output = tmp_path / "gh_output" + github_output.write_text("") + fake_binary(bindir, "terraform-bin", f"#!/bin/sh\necho '{json.dumps(PLAN)}'\n") + # Stands in for the setup-terraform wrapper: it echoes the plan AND appends it to + # $GITHUB_OUTPUT, exactly as core.setOutput('stdout', ...) does. + fake_binary( + bindir, + "terraform", + f"#!/bin/sh\necho 'stdout<> \"$GITHUB_OUTPUT\"\n" + f"echo '{json.dumps(PLAN)}' >> \"$GITHUB_OUTPUT\"\n" + f"echo '{json.dumps(PLAN)}'\n", + ) + monkeypatch.setenv("PATH", str(bindir)) + monkeypatch.setenv("GITHUB_OUTPUT", str(github_output)) + plan_file = tmp_path / "tfplan" + plan_file.write_bytes(b"binary") + + assert discover.terraform_show_json(str(plan_file)) == PLAN + assert github_output.read_text() == "", "the wrapper ran and leaked the plan into $GITHUB_OUTPUT" + + def test_invokes_show_json(self, tmp_path, monkeypatch): + bindir = tmp_path / "bin" + argv_log = tmp_path / "argv" + fake_binary( + bindir, + "terraform-bin", + f"#!/bin/sh\necho \"$@\" > '{argv_log}'\necho '{json.dumps(PLAN)}'\n", + ) + monkeypatch.setenv("PATH", str(bindir)) + plan_file = tmp_path / "tfplan" + plan_file.write_bytes(b"binary") + + discover.terraform_show_json(str(plan_file)) + + assert argv_log.read_text().startswith("show -json ") + + def test_a_wrapper_without_its_real_binary_is_refused(self, tmp_path, monkeypatch): + """ + TERRAFORM_CLI_PATH set but no terraform-bin anywhere means the only terraform on PATH is the + wrapper. Refuse rather than leak. + """ + bindir = tmp_path / "bin" + fake_binary(bindir, "terraform", "#!/bin/sh\nexit 0\n") + monkeypatch.setenv("PATH", str(bindir)) + monkeypatch.setenv("TERRAFORM_CLI_PATH", str(tmp_path / "toolcache")) + monkeypatch.delenv("TOFU_CLI_PATH", raising=False) + plan_file = tmp_path / "tfplan" + plan_file.write_bytes(b"binary") + + with pytest.raises(DiscoveryError, match="GITHUB_OUTPUT"): + discover.terraform_show_json(str(plan_file)) + + def test_a_failure_surfaces_stderr(self, tmp_path, monkeypatch): + bindir = tmp_path / "bin" + fake_binary(bindir, "terraform-bin", '#!/bin/sh\necho "Saved plan is stale" >&2\nexit 1\n') + monkeypatch.setenv("PATH", str(bindir)) + plan_file = tmp_path / "tfplan" + plan_file.write_bytes(b"binary") + + with pytest.raises(DiscoveryError, match="Saved plan is stale"): + discover.terraform_show_json(str(plan_file)) + + def test_non_json_output_does_not_echo_stdout(self, tmp_path, monkeypatch): + """On the wrapper path stdout would be the whole plan, so it must never reach the log.""" + bindir = tmp_path / "bin" + fake_binary(bindir, "terraform-bin", '#!/bin/sh\necho "AKIAIOSFODNN7EXAMPLE not json"\n') + monkeypatch.setenv("PATH", str(bindir)) + plan_file = tmp_path / "tfplan" + plan_file.write_bytes(b"binary") + + with pytest.raises(DiscoveryError) as excinfo: + discover.terraform_show_json(str(plan_file)) + + assert "AKIAIOSFODNN7EXAMPLE" not in str(excinfo.value) diff --git a/tests/platform/test_redact.py b/tests/platform/test_redact.py new file mode 100644 index 00000000..bc039b85 --- /dev/null +++ b/tests/platform/test_redact.py @@ -0,0 +1,1231 @@ +""" +Tests for plan/state redaction. + +This is the security-critical module: it is the only thing standing between a customer's secrets +and StackGuardian's storage. The tests assert on the *serialized bytes* wherever a leak would +matter, because a value nested somewhere unexpected still leaks even if the top-level shape looks +masked. +""" + +import json +import os +import sys + +import pytest + + +from tirith.platform import redact + +SECRET = "hunter2-this-must-never-leave-the-runner" + + +def test_slim_drops_prior_state_and_planned_values(): + """ + `planned_values` is the important one. It mirrors every resource's values in a second place + and carries NO sensitivity markers, so masking `resource_changes` alone leaves the same secret + in plaintext there. A real plan leaked a local_sensitive_file body through exactly this path. + """ + plan = { + "format_version": "1.2", + "terraform_version": "1.5.7", + "resource_changes": [], + "prior_state": {"values": {"secret": SECRET}}, + "planned_values": {"root_module": {"resources": [{"values": {"content": SECRET}}]}}, + } + + slimmed = redact.slim_plan(plan) + + assert "prior_state" not in slimmed + assert "planned_values" not in slimmed + assert slimmed["resource_changes"] == [] + assert slimmed["terraform_version"] == "1.5.7" + assert SECRET not in json.dumps(slimmed) + + +def test_planned_values_leak_is_closed_end_to_end(): + """The exact shape that leaked in QA: masked in resource_changes, plaintext in planned_values.""" + plan = { + "resource_changes": [ + { + "type": "local_sensitive_file", + "change": {"after": {"content": SECRET}, "after_sensitive": {"content": True}}, + } + ], + "planned_values": { + "root_module": {"resources": [{"type": "local_sensitive_file", "values": {"content": SECRET}}]} + }, + } + + redacted = redact.redact_plan(plan) + + assert SECRET not in json.dumps(redacted) + + +def test_configuration_is_kept_because_three_operations_read_it(): + """ + Dropping `configuration` would silently break direct_dependencies, direct_references and + provider_config: policies would stop finding what they look for rather than failing loudly. + """ + plan = { + "resource_changes": [], + "configuration": { + "root_module": {"resources": [{"address": "aws_vpc.main", "depends_on": ["aws_x.y"]}]}, + "provider_config": { + "aws": { + "name": "aws", + "full_name": "registry.terraform.io/hashicorp/aws", + "version_constraint": "~> 5.0", + "expressions": { + "region": {"constant_value": "eu-central-1"}, + "secret_key": {"constant_value": SECRET}, + "assume_role": {"role_arn": {"constant_value": SECRET}}, + }, + } + }, + }, + } + + slimmed = redact.slim_plan(plan) + aws = slimmed["configuration"]["provider_config"]["aws"] + + # What the provider_config operation reads survives ... + assert aws["full_name"] == "registry.terraform.io/hashicorp/aws" + assert aws["version_constraint"] == "~> 5.0" + assert aws["expressions"]["region"]["constant_value"] == "eu-central-1" + # ... and the reference graph the other two operations walk survives ... + assert slimmed["configuration"]["root_module"]["resources"][0]["depends_on"] == ["aws_x.y"] + # ... while hardcoded credentials do not. + assert "secret_key" not in aws["expressions"] + assert "assume_role" not in aws["expressions"] + assert SECRET not in json.dumps(slimmed) + + +def test_hcl_literals_are_scrubbed_from_resource_expressions(): + """ + The third instance of the `planned_values` pattern, caught in QA: a hardcoded value is masked + in `resource_changes` and sits in plaintext under + `configuration.root_module.resources[].expressions[].constant_value`, which carries no + sensitivity markers at all. + + Dropping it is lossless -- direct_references reads only `references`, direct_dependencies only + `depends_on`. + """ + plan = { + "resource_changes": [ + { + "type": "local_sensitive_file", + "change": {"after": {"content": SECRET}, "after_sensitive": {"content": True}}, + } + ], + "configuration": { + "root_module": { + "resources": [ + { + "address": "local_sensitive_file.creds", + "depends_on": ["null_resource.a"], + "expressions": { + "content": {"constant_value": SECRET}, + "filename": {"references": ["path.module"]}, + }, + } + ] + } + }, + } + + redacted = redact.redact_plan(plan) + expressions = redacted["configuration"]["root_module"]["resources"][0]["expressions"] + + assert SECRET not in json.dumps(redacted) + # The reference graph the operations walk survives ... + assert expressions["filename"]["references"] == ["path.module"] + assert redacted["configuration"]["root_module"]["resources"][0]["depends_on"] == ["null_resource.a"] + # ... the literal does not. + assert "constant_value" not in expressions["content"] + + +def test_nested_and_repeated_block_literals_are_scrubbed(): + """A block argument is a dict of expressions and a repeated block is a list of them.""" + plan = { + "resource_changes": [], + "configuration": { + "root_module": { + "resources": [ + { + "address": "aws_instance.web", + "expressions": { + "root_block_device": {"kms_key_id": {"constant_value": SECRET}}, + "ebs_block_device": [ + {"snapshot_id": {"constant_value": SECRET}}, + {"volume_id": {"references": ["aws_ebs_volume.a.id"]}}, + ], + }, + } + ] + } + }, + } + + redacted = redact.redact_plan(plan) + + assert SECRET not in json.dumps(redacted) + ebs = redacted["configuration"]["root_module"]["resources"][0]["expressions"]["ebs_block_device"] + assert ebs[1]["volume_id"]["references"] == ["aws_ebs_volume.a.id"] + + +def test_child_module_literals_are_scrubbed(): + plan = { + "resource_changes": [], + "configuration": { + "root_module": { + "module_calls": { + "db": { + "source": "./modules/db", + "expressions": {"password": {"constant_value": SECRET}}, + "module": { + "resources": [ + { + "address": "aws_db_instance.main", + "expressions": {"password": {"constant_value": SECRET}}, + } + ] + }, + } + } + } + }, + } + + redacted = redact.redact_plan(plan) + + assert SECRET not in json.dumps(redacted) + + +def test_variable_defaults_and_outputs_are_scrubbed(): + """A `default` on a sensitive variable is a literal in the configuration too.""" + plan = { + "resource_changes": [], + "configuration": { + "root_module": { + "variables": {"db_password": {"default": SECRET, "sensitive": True}}, + "outputs": {"conn": {"expression": {"constant_value": SECRET}}}, + } + }, + } + + redacted = redact.redact_plan(plan) + + assert SECRET not in json.dumps(redacted) + # The declaration itself survives; only the value goes. + assert redacted["configuration"]["root_module"]["variables"]["db_password"]["sensitive"] is True + + +def test_scrub_tolerates_a_provider_config_without_expressions(): + plan = {"resource_changes": [], "configuration": {"provider_config": {"null": {"name": "null"}}}} + + slimmed = redact.slim_plan(plan) + + assert slimmed["configuration"]["provider_config"]["null"] == {"name": "null"} + + +def test_redact_masks_marked_attributes(): + plan = { + "resource_changes": [ + { + "address": "aws_db_instance.main", + "type": "aws_db_instance", + "change": { + "actions": ["create"], + "before": None, + "after": {"identifier": "main", "password": SECRET, "port": 5432}, + "after_sensitive": {"password": True}, + }, + } + ] + } + + redacted = redact.redact_plan(plan) + after = redacted["resource_changes"][0]["change"]["after"] + + assert after["password"] == redact.SENTINEL + assert after["identifier"] == "main", "non-sensitive values must survive" + assert after["port"] == 5432 + assert SECRET not in json.dumps(redacted) + + +def test_redact_masks_a_whole_sensitive_subtree(): + """A marker of `true` above an object masks everything beneath it.""" + plan = { + "resource_changes": [ + { + "address": "aws_secretsmanager_secret_version.v", + "change": { + "after": {"secret_string": {"user": "admin", "pass": SECRET}}, + "after_sensitive": {"secret_string": True}, + }, + } + ] + } + + redacted = redact.redact_plan(plan) + + assert redacted["resource_changes"][0]["change"]["after"]["secret_string"] == redact.SENTINEL + assert SECRET not in json.dumps(redacted) + + +def test_redact_masks_inside_lists_positionally(): + plan = { + "resource_changes": [ + { + "change": { + "after": {"items": [{"k": "public"}, {"k": SECRET}]}, + "after_sensitive": {"items": [{}, {"k": True}]}, + } + } + ] + } + + redacted = redact.redact_plan(plan) + items = redacted["resource_changes"][0]["change"]["after"]["items"] + + assert items[0]["k"] == "public" + assert items[1]["k"] == redact.SENTINEL + assert SECRET not in json.dumps(redacted) + + +def test_redact_masks_before_as_well_as_after(): + """A destroy or update leaves the old secret in `before`; it leaks just as badly.""" + plan = { + "resource_changes": [ + { + "change": { + "actions": ["delete"], + "before": {"password": SECRET}, + "before_sensitive": {"password": True}, + "after": None, + } + } + ] + } + + redacted = redact.redact_plan(plan) + + assert redacted["resource_changes"][0]["change"]["before"]["password"] == redact.SENTINEL + assert SECRET not in json.dumps(redacted) + + +def test_redact_drops_root_variables_entirely(): + """ + The plan does not reliably mark which root variables were declared sensitive, so the only safe + assumption is that any of them might be. + """ + plan = {"resource_changes": [], "variables": {"db_password": {"value": SECRET}}} + + redacted = redact.redact_plan(plan) + + assert "variables" not in redacted + assert SECRET not in json.dumps(redacted) + + +def test_redact_masks_sensitive_output_changes(): + plan = { + "resource_changes": [], + "output_changes": { + "db_url": {"actions": ["create"], "after": SECRET, "sensitive": True}, + "region": {"actions": ["create"], "after": "eu-central-1", "sensitive": False}, + }, + } + + redacted = redact.redact_plan(plan) + + assert redacted["output_changes"]["db_url"]["after"] == redact.SENTINEL + assert redacted["output_changes"]["region"]["after"] == "eu-central-1" + assert SECRET not in json.dumps(redacted) + + +def test_redact_leaves_unmarked_values_alone(): + """ + Documents the known limitation honestly: terraform's markers are not exhaustive, so a secret + that arrives unmarked is NOT masked. Slimming and the variables drop limit the blast radius; + this test exists so the gap is visible rather than assumed away. + """ + plan = {"resource_changes": [{"change": {"after": {"password_from_locals": SECRET}, "after_sensitive": {}}}]} + + redacted = redact.redact_plan(plan) + + assert redacted["resource_changes"][0]["change"]["after"]["password_from_locals"] == SECRET + + +def test_redact_plan_tolerates_junk(): + assert redact.redact_plan({}) == {} + assert redact.redact_plan({"resource_changes": "not-a-list"})["resource_changes"] == "not-a-list" + assert redact.redact_plan([]) == [] + + +# --- state ------------------------------------------------------------------------------------- + + +def test_redact_state_masks_sensitive_outputs(): + state = { + "version": 4, + "outputs": { + "db_password": {"value": SECRET, "type": "string", "sensitive": True}, + "region": {"value": "eu-central-1", "type": "string"}, + }, + "resources": [], + } + + redacted = redact.redact_state(state) + + assert redacted["outputs"]["db_password"]["value"] == redact.SENTINEL + assert redacted["outputs"]["region"]["value"] == "eu-central-1" + assert SECRET not in json.dumps(redacted) + + +def test_redact_state_masks_sensitive_attributes(): + """ + The shape `terraform state pull` actually writes: each entry is a PATH -- a list of steps -- + not a single key. + + Captured verbatim from a real `local_sensitive_file`. The previous fixture here invented the + flat form, so this passed while real state was not masked at all: a list is neither a dict nor + a string, so every entry was skipped. + """ + state = { + "resources": [ + { + "type": "local_sensitive_file", + "name": "s", + "instances": [ + { + "attributes": {"id": "e590ef", "content": SECRET, "content_base64": SECRET}, + "sensitive_attributes": [ + [{"type": "get_attr", "value": "content_base64"}], + [{"type": "get_attr", "value": "content"}], + ], + } + ], + } + ] + } + + redacted = redact.redact_state(state) + attributes = redacted["resources"][0]["instances"][0]["attributes"] + + assert attributes["content"] == redact.SENTINEL + assert attributes["content_base64"] == redact.SENTINEL + assert attributes["id"] == "e590ef", "non-sensitive attributes must survive" + assert SECRET not in json.dumps(redacted) + + +def test_redact_state_masks_a_nested_attribute_path(): + """A path can descend through objects and list indices, not just name a top-level key.""" + state = { + "resources": [ + { + "instances": [ + { + "attributes": {"config": [{"token": SECRET, "url": "https://ok"}]}, + "sensitive_attributes": [ + [ + {"type": "get_attr", "value": "config"}, + {"type": "index", "value": 0}, + {"type": "get_attr", "value": "token"}, + ] + ], + } + ] + } + ] + } + + redacted = redact.redact_state(state) + config = redacted["resources"][0]["instances"][0]["attributes"]["config"][0] + + assert config["token"] == redact.SENTINEL + assert config["url"] == "https://ok" + + +def test_redact_state_does_not_mutate_the_input(): + """The caller still holds the original; masking must not reach back into it.""" + state = { + "resources": [ + { + "instances": [ + { + "attributes": {"password": SECRET}, + "sensitive_attributes": [[{"type": "get_attr", "value": "password"}]], + } + ] + } + ] + } + + redact.redact_state(state) + + assert state["resources"][0]["instances"][0]["attributes"]["password"] == SECRET + + +def test_redact_state_accepts_the_flat_get_attr_form(): + """Some providers and older state versions emit a single step rather than a path.""" + state = { + "resources": [ + { + "instances": [ + { + "attributes": {"password": SECRET}, + "sensitive_attributes": [{"type": "get_attr", "value": "password"}], + } + ] + } + ] + } + + redacted = redact.redact_state(state) + + assert redacted["resources"][0]["instances"][0]["attributes"]["password"] == redact.SENTINEL + + +def test_redact_state_accepts_bare_string_sensitive_attributes(): + """Older state versions write these as plain strings rather than objects.""" + state = {"resources": [{"instances": [{"attributes": {"secret": SECRET}, "sensitive_attributes": ["secret"]}]}]} + + redacted = redact.redact_state(state) + + assert redacted["resources"][0]["instances"][0]["attributes"]["secret"] == redact.SENTINEL + + +def test_redact_state_tolerates_junk(): + assert redact.redact_state({}) == {} + assert redact.redact_state({"resources": "nope"})["resources"] == "nope" + assert redact.redact_state({"outputs": None})["outputs"] is None + + +def test_count_redactions(): + document = {"a": redact.SENTINEL, "b": [redact.SENTINEL, "fine"], "c": {"d": redact.SENTINEL}} + + assert redact.count_redactions(document) == 3 + assert redact.count_redactions({"a": "fine"}) == 0 + + +# --- output_changes marker spellings ------------------------------------------------------------- +# +# These exist because a real plan slipped through: the code originally checked only a top-level +# `sensitive` key, but modern terraform emits `before_sensitive` / `after_sensitive` per side, so +# every sensitive output in a current plan went unmasked. + + +def test_output_change_masked_via_after_sensitive(): + """The spelling modern terraform actually uses.""" + plan = { + "resource_changes": [], + "output_changes": { + "db_url": {"actions": ["update"], "before": "old", "after": SECRET, "after_sensitive": True} + }, + } + + redacted = redact.redact_plan(plan) + + assert redacted["output_changes"]["db_url"]["after"] == redact.SENTINEL + assert SECRET not in json.dumps(redacted) + + +def test_output_change_masks_each_side_independently(): + """An output can become sensitive without having been so before, and vice versa.""" + plan = { + "resource_changes": [], + "output_changes": { + "rotated": { + "actions": ["update"], + "before": SECRET, + "after": "now-public", + "before_sensitive": True, + "after_sensitive": False, + } + }, + } + + redacted = redact.redact_plan(plan) + change = redacted["output_changes"]["rotated"] + + assert change["before"] == redact.SENTINEL + assert change["after"] == "now-public" + assert SECRET not in json.dumps(redacted) + + +def test_output_change_legacy_sensitive_key_masks_both_sides(): + plan = { + "resource_changes": [], + "output_changes": {"k": {"before": SECRET, "after": SECRET, "sensitive": True}}, + } + + redacted = redact.redact_plan(plan) + + assert redacted["output_changes"]["k"]["before"] == redact.SENTINEL + assert redacted["output_changes"]["k"]["after"] == redact.SENTINEL + + +def test_output_change_does_not_invent_absent_keys(): + """ + A create whose value is not yet known has no `after` at all (`after_unknown: true`). Adding a + sentinel would fabricate data the plan never carried, and would misrepresent the plan to any + policy reading it. + """ + plan = { + "resource_changes": [], + "output_changes": { + "pw": {"actions": ["create"], "before": None, "after_unknown": True, "after_sensitive": True} + }, + } + + redacted = redact.redact_plan(plan) + change = redacted["output_changes"]["pw"] + + assert "after" not in change + assert change["before"] is None + + +def test_unknown_create_values_are_simply_absent_from_the_plan(): + """ + Documents a property that made an earlier end-to-end test weaker than intended: for a create, + terraform does not know the value yet, so it is absent from `after` rather than present and + masked. Nothing leaks -- but a test that expects to see a sentinel here is testing nothing. + """ + plan = { + "resource_changes": [ + { + "type": "random_password", + "change": { + "actions": ["create"], + "after": {"length": 32}, + "after_unknown": {"result": True}, + "after_sensitive": {"result": True}, + }, + } + ] + } + + redacted = redact.redact_plan(plan) + after = redacted["resource_changes"][0]["change"]["after"] + + assert "result" not in after + assert redact.count_redactions(redacted) == 0 + + +def test_known_sensitive_value_at_plan_time_is_masked(): + """ + The case that DOES exercise marker-driven redaction: a hardcoded sensitive attribute is known + at plan time, so it really is in `after` and really must be replaced. + """ + plan = { + "resource_changes": [ + { + "type": "local_sensitive_file", + "change": { + "actions": ["create"], + "after": {"filename": "out.txt", "content": SECRET}, + "after_sensitive": {"content": True}, + }, + } + ] + } + + redacted = redact.redact_plan(plan) + + assert redacted["resource_changes"][0]["change"]["after"]["content"] == redact.SENTINEL + assert redacted["resource_changes"][0]["change"]["after"]["filename"] == "out.txt" + assert SECRET not in json.dumps(redacted) + + +# --- planned_values reconstruction ---------------------------------------------------------- + + +def _plan_with(resource_changes, **extra): + plan = {"format_version": "1.2", "terraform_version": "1.5.7", "resource_changes": resource_changes} + plan.update(extra) + return plan + + +def test_planned_values_is_rebuilt_so_infracost_and_checkov_have_something_to_read(): + """ + Both tools read planned_values and nothing else. Measured against infracost 0.10.27 with a + real key: the same t3.medium prices at $39.80 with this section and $0.00 without. + """ + out = redact.redact_plan( + _plan_with( + [ + { + "address": "aws_instance.app", + "mode": "managed", + "type": "aws_instance", + "name": "app", + "provider_name": "registry.terraform.io/hashicorp/aws", + "change": {"actions": ["create"], "after": {"instance_type": "t3.medium"}}, + } + ] + ) + ) + + resources = out["planned_values"]["root_module"]["resources"] + assert [r["address"] for r in resources] == ["aws_instance.app"] + assert resources[0]["values"]["instance_type"] == "t3.medium" + assert resources[0]["provider_name"] == "registry.terraform.io/hashicorp/aws" + + +def test_the_rebuilt_planned_values_carries_masked_values_not_raw_ones(): + """ + The whole reason terraform's own copy is dropped: it mirrors every value with no sensitivity + markers, so masking resource_changes leaves the secret in plaintext there. A real plan leaked + a local_sensitive_file body through exactly that path. This copy is derived post-masking. + """ + out = redact.redact_plan( + _plan_with( + [ + { + "address": "local_sensitive_file.creds", + "mode": "managed", + "type": "local_sensitive_file", + "name": "creds", + "change": { + "actions": ["create"], + "after": {"content": "hunter2", "filename": "/tmp/c"}, + "after_sensitive": {"content": True}, + }, + } + ], + planned_values={ + "root_module": { + "resources": [{"address": "local_sensitive_file.creds", "values": {"content": "hunter2"}}] + } + }, + ) + ) + + assert "hunter2" not in json.dumps(out) + values = out["planned_values"]["root_module"]["resources"][0]["values"] + assert values["content"] == redact.SENTINEL + assert values["filename"] == "/tmp/c", "non-sensitive attributes must survive" + + +def test_terraform_own_planned_values_is_never_passed_through(): + """It is replaced, not merged -- otherwise the unmarked original would leak straight through.""" + out = redact.redact_plan( + _plan_with( + [ + { + "address": "aws_instance.app", + "mode": "managed", + "type": "aws_instance", + "name": "app", + "change": {"actions": ["create"], "after": {"instance_type": "t3.medium"}}, + } + ], + planned_values={ + "root_module": { + "resources": [{"address": "ghost.resource", "values": {"secret": "leaked-from-original"}}] + } + }, + ) + ) + + assert "leaked-from-original" not in json.dumps(out) + assert [r["address"] for r in out["planned_values"]["root_module"]["resources"]] == ["aws_instance.app"] + + +def test_a_destroyed_resource_has_no_planned_value(): + """Nothing is planned to exist, so there is nothing to price or scan.""" + out = redact.redact_plan( + _plan_with( + [ + { + "address": "aws_instance.gone", + "mode": "managed", + "type": "aws_instance", + "name": "gone", + "change": {"actions": ["delete"], "before": {"instance_type": "m5.large"}, "after": None}, + } + ] + ) + ) + + assert "planned_values" not in out + assert out["resource_changes"], "the destroy is still a change policies evaluate" + + +def test_a_replacement_is_planned_because_it_ends_up_existing(): + out = redact.redact_plan( + _plan_with( + [ + { + "address": "aws_instance.app", + "mode": "managed", + "type": "aws_instance", + "name": "app", + "change": {"actions": ["delete", "create"], "after": {"instance_type": "t3.large"}}, + } + ] + ) + ) + + assert out["planned_values"]["root_module"]["resources"][0]["values"]["instance_type"] == "t3.large" + + +def test_module_resources_are_grouped_under_child_modules(): + out = redact.redact_plan( + _plan_with( + [ + { + "address": "aws_instance.app", + "mode": "managed", + "type": "aws_instance", + "name": "app", + "change": {"actions": ["create"], "after": {"instance_type": "t3.medium"}}, + }, + { + "address": "module.db.aws_instance.replica", + "module_address": "module.db", + "mode": "managed", + "type": "aws_instance", + "name": "replica", + "change": {"actions": ["create"], "after": {"instance_type": "m5.large"}}, + }, + ] + ) + ) + + root = out["planned_values"]["root_module"] + assert [r["address"] for r in root["resources"]] == ["aws_instance.app"] + assert [m["address"] for m in root["child_modules"]] == ["module.db"] + assert root["child_modules"][0]["resources"][0]["address"] == "module.db.aws_instance.replica" + + +def test_child_modules_is_absent_when_there_are_none(): + out = redact.redact_plan( + _plan_with( + [ + { + "address": "aws_instance.app", + "mode": "managed", + "type": "aws_instance", + "name": "app", + "change": {"actions": ["create"], "after": {"instance_type": "t3.medium"}}, + } + ] + ) + ) + + assert "child_modules" not in out["planned_values"]["root_module"] + + +def test_an_empty_plan_gets_no_planned_values(): + assert "planned_values" not in redact.redact_plan(_plan_with([])) + + +# --- resource_drift and configuration literals --------------------------------------------------- + + +def test_resource_drift_is_masked_like_resource_changes(): + """ + resource_drift has the identical shape and the identical sensitivity markers, and terraform + emits it whenever a refresh finds drift. Masking resource_changes and leaving this alone shipped + the same secret one key away -- the planned_values failure a third time. + """ + plan = { + "format_version": "1.2", + "resource_drift": [ + { + "address": "aws_secretsmanager_secret_version.db", + "type": "aws_secretsmanager_secret_version", + "change": { + "actions": ["update"], + "before": {"secret_string": "hunter2-before"}, + "after": {"secret_string": "hunter2-after"}, + "before_sensitive": {"secret_string": True}, + "after_sensitive": {"secret_string": True}, + }, + } + ], + } + + out = redact.redact_plan(plan) + drift = out["resource_drift"][0]["change"] + + assert drift["before"]["secret_string"] == redact.SENTINEL + assert drift["after"]["secret_string"] == redact.SENTINEL + assert "hunter2-before" not in json.dumps(out) + assert "hunter2-after" not in json.dumps(out) + + +def test_provisioner_literals_are_scrubbed_from_configuration(): + """ + A provisioner carries its own expressions one level below the resource's, and a connection block + is exactly where a password gets written literally. Scrubbing only the resource's own + expressions left these verbatim -- and configuration ships even with `source-dir: ""`. + """ + plan = { + "format_version": "1.2", + "configuration": { + "root_module": { + "resources": [ + { + "address": "aws_instance.app", + "expressions": {"ami": {"constant_value": "ami-123"}}, + "provisioners": [ + { + "type": "remote-exec", + "expressions": { + "inline": {"constant_value": ["echo s3cr3t-inline"]}, + "connection": {"password": {"constant_value": "s3cr3t-conn"}}, + }, + } + ], + } + ] + } + }, + } + + out = json.dumps(redact.redact_plan(plan)) + + assert "s3cr3t-conn" not in out + assert "s3cr3t-inline" not in out + + +def test_module_call_arguments_are_dropped_even_without_an_inlined_module(): + """ + A module sourced from a registry or a git ref carries no inlined `module` body, which is the + common case -- and its arguments are literals either way. + """ + plan = { + "format_version": "1.2", + "configuration": { + "root_module": { + "module_calls": { + "db": { + "source": "terraform-aws-modules/rds/aws", + "expressions": {"password": {"constant_value": "s3cr3t-mod"}}, + } + } + } + }, + } + + assert "s3cr3t-mod" not in json.dumps(redact.redact_plan(plan)) + + +def test_a_show_json_state_is_masked_not_passed_through(): + """ + The leak an end-to-end run found. `terraform show -json ` is the natural way to get a + readable state, and its shape nests resources under values.root_module with a parallel + sensitive_values tree -- nothing like the raw state this function was written for. It returned + the document unchanged: no error, no warning, every attribute in plaintext. + """ + document = { + "format_version": "1.0", + "values": { + "root_module": { + "resources": [ + { + "address": "aws_db_instance.main", + "type": "aws_db_instance", + "values": {"identifier": "prod-db", "password": "hunter2"}, + "sensitive_values": {"password": True}, + } + ], + "child_modules": [ + { + "address": "module.net", + "resources": [ + { + "address": "module.net.aws_secretsmanager_secret_version.k", + "values": {"secret_string": "hunter3"}, + "sensitive_values": {"secret_string": True}, + } + ], + } + ], + }, + "outputs": {"db_url": {"value": "postgres://hunter4@host", "sensitive": True}}, + }, + } + + out = redact.redact_state(document) + blob = json.dumps(out) + + assert out["values"]["root_module"]["resources"][0]["values"]["password"] == redact.SENTINEL + # A module's resources are nested, not flattened -- masking only the root would miss them. + assert ( + out["values"]["root_module"]["child_modules"][0]["resources"][0]["values"]["secret_string"] == redact.SENTINEL + ) + assert out["values"]["outputs"]["db_url"]["value"] == redact.SENTINEL + for secret in ("hunter2", "hunter3", "hunter4"): + assert secret not in blob, secret + + +def test_the_raw_state_shape_still_works(): + """The shape this function was written for must keep working alongside the new one.""" + document = { + "version": 4, + "resources": [ + { + "type": "aws_db_instance", + "instances": [{"attributes": {"password": "hunter2"}, "sensitive_attributes": ["password"]}], + } + ], + "outputs": {"token": {"value": "hunter5", "sensitive": True}}, + } + + out = redact.redact_state(document) + + assert out["resources"][0]["instances"][0]["attributes"]["password"] == redact.SENTINEL + assert out["outputs"]["token"]["value"] == redact.SENTINEL + + +# --- provider-computed mirrors: the markers are not enough ----------------------------------------- +# +# Terraform does not propagate sensitivity into attributes a provider computes from a sensitive one. +# An aws_instance with a secret in `tags` is marked `after_sensitive.tags.Password = true`, while +# `after_sensitive.tags_all` comes back `{}` even though `tags_all` holds the identical plaintext. +# Every AWS resource with tags has `tags_all`, so that one gap leaks any secret used in a tag. +# +# Found by an E2E that downloaded the uploaded bundle and grepped it. The unit suite was green +# throughout, because it asserted the markers were honoured -- and they were. + + +def _plan_with_tags_all(): + return { + "format_version": "1.2", + "resource_changes": [ + { + "address": "aws_instance.app", + "type": "aws_instance", + "change": { + "actions": ["create"], + "after": { + "instance_type": "t3.micro", + "tags": {"Name": "keep-me", "Password": "hunter2-plan-secret"}, + "tags_all": {"Name": "keep-me", "Password": "hunter2-plan-secret"}, + }, + "after_sensitive": {"tags": {"Password": True}, "tags_all": {}}, + }, + } + ], + } + + +def test_a_secret_mirrored_into_an_unmarked_attribute_is_still_masked(): + masked = redact.redact_plan(_plan_with_tags_all()) + + assert "hunter2-plan-secret" not in json.dumps(masked), "tags_all leaked the secret terraform marked in tags" + + +def test_the_sweep_does_not_mangle_values_that_were_never_sensitive(): + """Over-redaction would corrupt the document the policies read, which is its own kind of failure.""" + masked = redact.redact_plan(_plan_with_tags_all()) + after = masked["resource_changes"][0]["change"]["after"] + + assert after["instance_type"] == "t3.micro" + assert after["tags"]["Name"] == "keep-me" + assert after["tags_all"]["Name"] == "keep-me" + + +def test_a_sensitive_root_variable_is_swept_out_of_the_resources_too(): + """ + The variable block is dropped wholesale, but its value routinely reappears in an unmarked + attribute -- so the value has to be collected before it is dropped. + """ + plan = { + "variables": {"db_password": {"value": "hunter2-plan-secret"}}, + "resource_changes": [ + { + "address": "aws_instance.app", + "type": "aws_instance", + "change": { + "actions": ["create"], + "after": {"tags_all": {"Password": "hunter2-plan-secret"}}, + "after_sensitive": {}, + }, + } + ], + } + + masked = redact.redact_plan(plan) + + assert "variables" not in masked + assert "hunter2-plan-secret" not in json.dumps(masked) + + +def test_a_very_short_sensitive_value_is_not_swept(): + """ + The sweep matches exact strings everywhere, so a two-character secret would also match ids and + regions and mangle the plan. Leaking a two-character value is the lesser harm against breaking + every policy on the document. + """ + plan = { + "resource_changes": [ + { + "address": "aws_instance.app", + "type": "aws_instance", + "change": { + "actions": ["create"], + "after": {"tags": {"P": "ab"}, "region": "ab", "instance_type": "t3.micro"}, + "after_sensitive": {"tags": {"P": True}}, + }, + } + ], + } + + masked = redact.redact_plan(plan) + after = masked["resource_changes"][0]["change"]["after"] + + assert after["tags"]["P"] == redact.SENTINEL, "the marked value is still masked by the marker" + assert after["region"] == "ab", "but an unrelated two-character value must survive" + + +# --- state: provider-computed mirrors ----------------------------------------------------------- +# +# From a penetration test. `redact_plan` already swept the plaintext of every marked value across the +# whole document to catch computed mirrors; `redact_state` did not, so a secret in a tag was masked at +# `tags.Password` and shipped in cleartext at `tags_all.Password`. +# +# State is the worse place for this hole than a plan: it carries every attribute of every resource, and +# the bundle it is uploaded in is retained indefinitely. Neither existing state test had an unmarked +# mirror attribute, which is why both passed with the leak present. + +TAG_SECRET = "hunter2-tag-secret" + + +def _raw_state_with_tags_all(): + """Raw `terraform state pull`: sensitivity is a list of attribute paths.""" + return { + "version": 4, + "resources": [ + { + "type": "aws_instance", + "name": "app", + "instances": [ + { + "attributes": { + "tags": {"Password": TAG_SECRET}, + # The provider's computed mirror. Same plaintext, named by nothing. + "tags_all": {"Password": TAG_SECRET}, + "region": "us-east-1", + }, + "sensitive_attributes": [ + [{"type": "get_attr", "value": "tags"}, {"type": "get_attr", "value": "Password"}] + ], + } + ], + } + ], + } + + +def _show_json_state_with_tags_all(): + """`terraform show -json `: sensitivity is a parallel marker tree, as in a plan.""" + return { + "format_version": "1.0", + "values": { + "root_module": { + "resources": [ + { + "address": "aws_instance.app", + "values": { + "tags": {"Password": TAG_SECRET}, + "tags_all": {"Password": TAG_SECRET}, + "region": "us-east-1", + }, + # tags_all is present and empty -- terraform marks nothing in it. + "sensitive_values": {"tags": {"Password": True}, "tags_all": {}}, + } + ] + } + }, + } + + +@pytest.mark.parametrize( + "build, read", + [ + (_raw_state_with_tags_all, lambda o: o["resources"][0]["instances"][0]["attributes"]), + (_show_json_state_with_tags_all, lambda o: o["values"]["root_module"]["resources"][0]["values"]), + ], + ids=["raw", "show-json"], +) +def test_a_secret_mirrored_into_an_unmarked_state_attribute_is_still_masked(build, read): + out = redact.redact_state(build()) + attributes = read(out) + + assert attributes["tags"]["Password"] == redact.SENTINEL + assert attributes["tags_all"]["Password"] == redact.SENTINEL + # The whole-document assertion is the one that matters: the mirror is only the case we know about. + assert TAG_SECRET not in json.dumps(out) + + +@pytest.mark.parametrize( + "build, read", + [ + (_raw_state_with_tags_all, lambda o: o["resources"][0]["instances"][0]["attributes"]), + (_show_json_state_with_tags_all, lambda o: o["values"]["root_module"]["resources"][0]["values"]), + ], + ids=["raw", "show-json"], +) +def test_the_state_sweep_does_not_redact_unrelated_values(build, read): + """A sweep that masks by value will over-mask if it is not bounded. `region` is not a secret.""" + out = redact.redact_state(build()) + + assert read(out)["region"] == "us-east-1" + + +def test_a_sensitive_state_output_is_swept_out_of_a_resource_attribute(): + """ + An output's plaintext is discarded when the output is masked, so nothing else knew it was a secret -- + and the same value sitting in an ordinary attribute stayed in cleartext. + """ + state = { + "version": 4, + "outputs": {"db_password": {"value": TAG_SECRET, "sensitive": True}}, + "resources": [ + { + "type": "aws_db_instance", + "instances": [{"attributes": {"password_copy": TAG_SECRET, "engine": "postgres"}}], + } + ], + } + + out = redact.redact_state(state) + + assert out["outputs"]["db_password"]["value"] == redact.SENTINEL + assert out["resources"][0]["instances"][0]["attributes"]["password_copy"] == redact.SENTINEL + assert out["resources"][0]["instances"][0]["attributes"]["engine"] == "postgres" + assert TAG_SECRET not in json.dumps(out) + + +def test_a_short_state_secret_is_not_swept(): + """ + The length floor exists so masking one short value does not redact every id, region and short + string that happens to equal it. Same bound as the plan sweep. + """ + state = { + "version": 4, + "resources": [ + { + "type": "aws_instance", + "instances": [ + { + "attributes": {"tags": {"Env": "dev"}, "tags_all": {"Env": "dev"}, "stage": "dev"}, + "sensitive_attributes": [ + [{"type": "get_attr", "value": "tags"}, {"type": "get_attr", "value": "Env"}] + ], + } + ], + } + ], + } + + out = redact.redact_state(state) + attributes = out["resources"][0]["instances"][0]["attributes"] + + # Masked where it is marked, and left alone everywhere else. + assert attributes["tags"]["Env"] == redact.SENTINEL + assert attributes["stage"] == "dev" diff --git a/tests/platform/test_regions.py b/tests/platform/test_regions.py new file mode 100644 index 00000000..f5fdbebf --- /dev/null +++ b/tests/platform/test_regions.py @@ -0,0 +1,169 @@ +""" +Tests for the region table and URL resolution. + +The failure this replaces: `--api-url` and `--dashboard-url` were independent, so overriding only +the API left every run link in every PR comment pointing at the wrong environment -- which reads as +a broken integration rather than a misconfiguration. +""" + +import pytest + +from tirith.platform import regions + +EU_API = "https://api.app.stackguardian.io/api/v1" +EU_APP = "https://app.stackguardian.io" +US_API = "https://api.us.stackguardian.io/api/v1" +US_APP = "https://us.stackguardian.io" + + +class TestTable: + def test_two_production_regions(self): + assert regions.REGION_IDS == ("eu", "us") + + def test_eu_is_the_default(self): + assert regions.DEFAULT_REGION_ID == "eu" + + @pytest.mark.parametrize( + "region_id, api_base, app_base", + [ + ("eu", "https://api.app.stackguardian.io", EU_APP), + ("us", "https://api.us.stackguardian.io", US_APP), + ], + ) + def test_region_pairs(self, region_id, api_base, app_base): + region = regions.by_id(region_id) + assert region.api_base == api_base + assert region.app_base == app_base + + def test_api_bases_omit_the_api_path(self): + """Matches Raycast, sg-cli and the terraform provider; normalize_api_url adds it back.""" + for region in regions.REGIONS: + assert not region.api_base.endswith("/api/v1") + + def test_unknown_region_raises_and_names_the_valid_ones(self): + """ + Deliberately not Raycast's "fall back to the first region": a typo would silently point a US + org at production EU, and the only symptom would be an unexplainable auth error. + """ + with pytest.raises(ValueError) as excinfo: + regions.by_id("uss") + assert "eu" in str(excinfo.value) + assert "us" in str(excinfo.value) + + +class TestNormalizeApiUrl: + @pytest.mark.parametrize( + "given", + [ + "https://api.app.stackguardian.io", + "https://api.app.stackguardian.io/", + "https://api.app.stackguardian.io/api/v1", + "https://api.app.stackguardian.io/api/v1/", + ], + ) + def test_both_spellings_converge(self, given): + """ + sg-cli's SG_BASE_URL omits /api/v1 and tirith's has always included it, so a value exported + for one produced 404s from the other. + """ + assert regions.normalize_api_url(given) == EU_API + + def test_an_empty_value_stays_empty(self): + assert regions.normalize_api_url("") == "" + assert regions.normalize_api_url(None) == "" + + def test_a_self_hosted_host_is_left_alone_apart_from_the_suffix(self): + assert regions.normalize_api_url("https://api.siemens-ag.stackguardian.io") == ( + "https://api.siemens-ag.stackguardian.io/api/v1" + ) + + +class TestByApiUrl: + @pytest.mark.parametrize("given", ["https://api.us.stackguardian.io", US_API]) + def test_matches_with_or_without_the_suffix(self, given): + assert regions.by_api_url(given).id == "us" + + def test_returns_none_for_an_unknown_host(self): + assert regions.by_api_url("https://api.siemens-ag.stackguardian.io") is None + + +class TestResolve: + def test_defaults_to_eu(self): + api, dashboard, warnings = regions.resolve() + assert (api, dashboard) == (EU_API, EU_APP) + assert warnings == [] + + def test_region_sets_both_urls(self): + api, dashboard, warnings = regions.resolve(region_id="us") + assert (api, dashboard) == (US_API, US_APP) + assert warnings == [] + + def test_explicit_urls_win_over_the_default(self): + api, dashboard, _w = regions.resolve( + api_url="https://api.self-hosted.example", dashboard_url="https://self-hosted.example" + ) + assert api == "https://api.self-hosted.example/api/v1" + assert dashboard == "https://self-hosted.example" + + @pytest.mark.parametrize( + "kwargs", + [ + {"api_url": "https://api.self-hosted.example"}, + {"dashboard_url": "https://self-hosted.example"}, + {"api_url": "https://api.self-hosted.example", "dashboard_url": "https://self-hosted.example"}, + ], + ) + def test_region_with_an_explicit_url_is_an_error(self, kwargs): + """They set the same thing; silently picking one would hide the contradiction.""" + with pytest.raises(ValueError, match="cannot be combined"): + regions.resolve(region_id="us", **kwargs) + + def test_an_api_url_for_a_known_region_infers_its_dashboard(self): + """ + The footgun the whole module exists for: this used to leave run links on the EU dashboard + for a US org. + """ + api, dashboard, warnings = regions.resolve(api_url="https://api.us.stackguardian.io") + assert api == US_API + assert dashboard == US_APP + assert warnings == [] + + def test_an_unknown_api_url_without_a_dashboard_warns(self): + api, dashboard, warnings = regions.resolve(api_url="https://api.self-hosted.example") + assert api == "https://api.self-hosted.example/api/v1" + assert dashboard == EU_APP + assert len(warnings) == 1 + assert "--dashboard-url" in warnings[0] + + +class TestResolveFromEnvironment: + def test_sg_region_is_honoured(self): + api, dashboard, _w = regions.resolve(env={"SG_REGION": "us"}) + assert (api, dashboard) == (US_API, US_APP) + + def test_sg_base_url_without_the_suffix_is_normalized(self): + api, _d, _w = regions.resolve(env={"SG_BASE_URL": "https://api.us.stackguardian.io"}) + assert api == US_API + + def test_sg_base_url_infers_the_dashboard_too(self): + _api, dashboard, _w = regions.resolve(env={"SG_BASE_URL": "https://api.us.stackguardian.io"}) + assert dashboard == US_APP + + def test_an_explicit_flag_beats_the_environment(self): + api, _d, _w = regions.resolve(api_url="https://api.us.stackguardian.io", env={"SG_BASE_URL": "https://x"}) + assert api == US_API + + def test_a_region_flag_beats_a_url_environment(self): + api, dashboard, warnings = regions.resolve(region_id="us", env={"SG_BASE_URL": "https://x"}) + assert (api, dashboard) == (US_API, US_APP) + assert warnings == [] + + def test_a_url_environment_beats_sg_region_with_a_warning(self): + """ + Not an error: the environment is inherited config the caller may not control, and failing a + CI run over a contradiction they did not write would be unhelpful. + """ + api, _d, warnings = regions.resolve(env={"SG_REGION": "eu", "SG_BASE_URL": "https://api.us.stackguardian.io"}) + assert api == US_API + assert len(warnings) == 1 + assert "SG_REGION" in warnings[0] diff --git a/tests/platform/test_report.py b/tests/platform/test_report.py new file mode 100644 index 00000000..eb03fbd9 --- /dev/null +++ b/tests/platform/test_report.py @@ -0,0 +1,708 @@ +""" +Tests for verdict computation and comment rendering. + +The verdict mapping is the part worth pinning hardest: every path that does not produce a real +"everything passed" must stay distinguishable from one that does, and must never map to a green +required check. +""" + +import os +import re +import sys + +import pytest + + +from tirith.platform import report as render + + +def _results(result="FAIL", **rule_overrides): + rule = { + "rule_name": "ingress-cidr", + "result": result, + "evaluations": { + "fails": [ + { + "id": "check1", + "result": [ + { + "passed": False, + "message": "`0.0.0.0/0` is contained in `cidr_blocks`", + "meta": {"address": "module.net.aws_security_group.web"}, + } + ], + } + ] + }, + } + rule.update(rule_overrides) + return {"no-public-ingress": [rule]} + + +# --- summarize --------------------------------------------------------------------------------- + + +def test_summarize_counts_and_extracts_detail(): + counts, findings = render.summarize(_results()) + + assert counts["FAIL"] == 1 + assert findings[0]["policy_id"] == "no-public-ingress" + assert findings[0]["messages"] == ["`0.0.0.0/0` is contained in `cidr_blocks`"] + assert findings[0]["resources"] == ["module.net.aws_security_group.web"] + + +def test_summarize_counts_skipped_separately_from_passed(): + """Reporting a skipped control as passing would be a quiet inaccuracy.""" + counts, findings = render.summarize({"p": [{"rule_name": "r", "skip": True}]}) + + assert counts["SKIPPED"] == 1 + assert counts["PASS"] == 0 + assert findings[0]["result"] == "SKIPPED" + + +def test_summarize_surfaces_engine_errors_distinctly(): + """ + A malformed policy must not read as a policy violation. Prefixing makes it obvious in the + comment that the engine, not the infrastructure, is the problem. + """ + results = {"p": [{"rule_name": "r", "result": "FAIL", "evaluations": {"fails": [{"exec_err": "bad op"}]}}]} + + _, findings = render.summarize(results) + + assert findings[0]["messages"] == ["engine: bad op"] + + +def test_summarize_handles_providers_without_resource_addresses(): + """Only terraform_plan populates meta; json/kubernetes set it to None.""" + results = { + "p": [ + { + "rule_name": "r", + "result": "FAIL", + "evaluations": {"fails": [{"id": "c", "result": [{"message": "no", "meta": None}]}]}, + } + ] + } + + _, findings = render.summarize(results) + + assert findings[0]["resources"] == [] + assert findings[0]["messages"] == ["no"] + + +def test_summarize_tolerates_empty_and_none(): + assert render.summarize(None)[0]["FAIL"] == 0 + assert render.summarize({})[1] == [] + + +# --- verdict ----------------------------------------------------------------------------------- + + +def test_verdict_failed_when_any_policy_fails(): + counts, _ = render.summarize(_results("FAIL")) + assert render.verdict(counts, "COMPLETED") == "failed" + + +def test_verdict_warned_for_a_warning(): + counts, _ = render.summarize(_results("WARN")) + assert render.verdict(counts, "COMPLETED") == "warned" + + +def test_verdict_approval_required_warns_rather_than_gating(): + """ + A rule result of APPROVAL_REQUIRED means its author wrote `onFail: APPROVAL_REQUIRED`. For these + runs there is nothing to approve: the step exits 0, the run reaches COMPLETED, and an approval + is only ever engaged on exit 11 and never on the last step -- of which a policy-only run has + exactly one. So it warns, deliberately, until a real gate exists. + """ + counts, _ = render.summarize(_results("APPROVAL_REQUIRED")) + + assert render.verdict(counts, "COMPLETED") == "warned" + + +def test_verdict_failed_outranks_approval_required(): + """A hard failure is the more urgent signal when a run has both.""" + counts = {"FAIL": 1, "APPROVAL_REQUIRED": 1} + + assert render.verdict(counts, "COMPLETED") == "failed" + + +def test_verdict_passed_only_when_a_policy_actually_passed(): + counts, _ = render.summarize(_results("PASS")) + assert render.verdict(counts, "COMPLETED") == "passed" + + +def test_verdict_errored_for_a_non_completed_run(): + """An ERRORED or CANCELLED run produced no verdict; that is not a pass.""" + counts, _ = render.summarize(_results("PASS")) + for status in ("ERRORED", "CANCELLED", "RUNNING", None): + assert render.verdict(counts, status) == "errored", status + + +def test_verdict_distinguishes_no_policies_from_passed(): + """ + A run with nothing in scope is reported as such rather than as a clean bill of health -- the + most likely cause is a policy scoped to the wrong workflow group. + """ + assert render.verdict({}, "COMPLETED") == "no-policies" + + +def test_a_run_paused_by_the_platform_warns_when_it_produced_results(): + """ + A run resting at APPROVAL_REQUIRED evaluated something before it paused. Reporting it as + `errored` would blame the tool for a working evaluation -- and the poller stops there rather + than spinning to its timeout. + """ + counts, _ = render.summarize(_results("PASS")) + + assert render.verdict(counts, "APPROVAL_REQUIRED") == "warned" + + +def test_a_run_paused_before_it_evaluated_anything_is_an_error(): + """ + The one thing that must never happen: green, or even amber, for a run that produced no verdict. + A paused run with no results has not evaluated the code. + """ + assert render.verdict({}, "APPROVAL_REQUIRED") == "errored" + + +# --- rendering --------------------------------------------------------------------------------- + + +def test_markdown_starts_with_the_marker_when_one_is_given(): + """ + The marker is opaque to this module -- GitHub's sticky-comment marker is one caller's choice -- + but when supplied it must be line 1, so the caller can find the document again. + """ + marker = "[//]: <> (tirith-comment, tag=envs-prod)" + body = render.render_markdown(_results(), "COMPLETED", "https://app.example/run", marker=marker) + + assert body.split("\n")[0] == marker + + +def test_markdown_has_no_marker_line_by_default(): + """This module is VCS-agnostic: nothing is prepended unless the caller asks for it.""" + body = render.render_markdown(_results(), "COMPLETED", "https://app.example/run") + + assert not body.startswith("[//]") + assert body.lstrip().startswith("## ") + + +def test_comment_includes_table_detail_and_run_link(): + body = render.render_markdown(_results(), "COMPLETED", "https://app.example/run") + + assert "| Policy | Rule | Resource |" in body + assert "`no-public-ingress`" in body + assert "`0.0.0.0/0` is contained in `cidr_blocks`" in body + assert "module.net.aws_security_group.web" in body + assert "https://app.example/run" in body + + +def test_comment_explains_an_errored_run(): + body = render.render_markdown({}, "ERRORED", "https://app.example/run") + + assert "could not evaluate" in body.lower() + assert "ERRORED" in body + + +def test_comment_truncates_below_the_github_limit_keeping_the_table(): + """ + GitHub rejects a body over 65536 characters with a 422. Detail sections go first; the summary + table is what a reviewer scans, so it must survive. + """ + results = { + f"policy-{i}": [ + { + "rule_name": f"rule-{i}", + "result": "FAIL", + "evaluations": { + "fails": [ + { + "id": f"check-{j}", + "result": [ + { + "message": "x" * 400, + "meta": {"address": f"aws_instance.i{j}"}, + } + ], + } + for j in range(20) + ] + }, + } + ] + for i in range(60) + } + + body = render.render_markdown(results, "COMPLETED", "https://app.example/run", limit=20000) + + assert len(body) <= 20000 + assert "| Policy | Rule | Resource |" in body, "the summary table must survive truncation" + assert "more finding" in body or "truncated" in body + + +def test_strip_marker_removes_it_for_targets_that_have_no_use_for_it(): + """A check-run summary, for instance: the marker only means something on an issue comment.""" + marker = "[//]: <> (tirith-comment, tag=default)" + body = render.render_markdown(_results(), "COMPLETED", "https://app.example/run", marker=marker) + + summary = render.strip_marker(body) + + assert "[//]: <>" not in summary + assert "no-public-ingress" in summary + + +def test_headline_reports_each_nonzero_bucket(): + counts = {"FAIL": 2, "WARN": 1, "APPROVAL_REQUIRED": 3, "PASS": 9, "SKIPPED": 1} + + assert render.headline(counts, "failed") == "Tirith — 2 failed, 3 need approval, 1 warned, 9 passed, 1 skipped" + + +# --- cost line ---------------------------------------------------------------------------------- + + +def test_cost_line_shows_the_monthly_total(): + assert "39.80 USD" in "\n".join(render.render_cost({"totalMonthlyCost": "39.8", "currency": "USD"})) + + +def test_cost_line_shows_the_delta_from_this_change(): + """Infracost fills the diff from the plan's prior state -- the number a reviewer wants.""" + line = "\n".join(render.render_cost({"totalMonthlyCost": "120.5", "diffTotalMonthlyCost": "39.8"})) + + assert "120.50" in line + assert "+39.80 from this change" in line + + +def test_a_cost_decrease_reads_as_a_decrease(): + line = "\n".join(render.render_cost({"totalMonthlyCost": "10", "diffTotalMonthlyCost": "-5.25"})) + + assert "−5.25 from this change" in line + + +def test_a_zero_delta_is_omitted_rather_than_shown_as_plus_zero(): + line = "\n".join(render.render_cost({"totalMonthlyCost": "10", "diffTotalMonthlyCost": "0"})) + + assert "from this change" not in line + + +def test_a_zero_cost_is_still_reported(): + """Silence would be indistinguishable from 'this change costs nothing'.""" + assert "0.00" in "\n".join(render.render_cost({"totalMonthlyCost": "0"})) + + +def test_a_failed_estimate_says_so(): + line = "\n".join(render.render_cost({"error": "failed to perform infrastructure cost estimation"})) + + assert "unavailable" in line + + +def test_no_estimate_renders_nothing(): + assert render.render_cost(None) == [] + assert render.render_cost({}) == [] + + +def test_the_cost_appears_in_the_comment_body(): + body = render.render_markdown( + {"p": [{"rule_name": "r", "result": "PASS"}]}, + "COMPLETED", + "https://dash.example/run", + cost_breakdown={"totalMonthlyCost": "39.8", "currency": "USD"}, + ) + + assert "39.80 USD" in body + + +def test_the_cost_survives_truncation_of_a_long_findings_list(): + """A wall of findings must not push the cost line out of the comment.""" + results = { + f"policy-{i}": [ + { + "rule_name": f"rule-{i}", + "result": "FAIL", + "evaluations": {"fails": [{"result": [{"message": "x" * 400}]}]}, + } + ] + for i in range(60) + } + + body = render.render_markdown( + results, + "COMPLETED", + "https://dash.example/run", + limit=3000, + cost_breakdown={"totalMonthlyCost": "39.8"}, + ) + + assert len(body) <= 3000 + assert "39.80" in body + + +# --- checkov findings --------------------------------------------------------------------------- + + +def _checkov_rule(fails): + return { + "rule_name": "Policy-Rule-1", + "source_config_kind": "SG_INTERNAL_P2", + "result": "FAIL", + "evaluations": {"fails": fails}, + } + + +def test_checkov_findings_are_rendered(): + """ + Checkov entries are {"description", "keys"}, not tirith's list under "result". Reading only the + tirith shape rendered a dozen real findings as an empty
block -- in the one place a + reviewer looks. Taken verbatim from QA run iqkxb26uzi1n. + """ + body = render.render_markdown( + { + "best-practices": [ + _checkov_rule( + [ + { + "description": "Ensure that detailed monitoring is enabled for EC2 instances", + "keys": ["aws_instance.app.monitoring"], + }, + ] + ) + ] + }, + "COMPLETED", + "https://dash.example/run", + ) + + assert "Ensure that detailed monitoring is enabled for EC2 instances" in body + + +def test_a_checkov_key_is_reduced_to_its_resource_address(): + """The attribute suffix is what the check inspected; the address is what a reviewer navigates by.""" + _messages, resources = render._extract_detail( + _checkov_rule( + [ + { + "description": "Ensure S3 buckets are encrypted", + "keys": ["aws_s3_bucket.data.rule.apply_server_side_encryption_by_default.sse_algorithm"], + }, + ] + ) + ) + + assert resources == ["aws_s3_bucket.data"] + + +def test_repeated_keys_on_one_resource_are_listed_once(): + _messages, resources = render._extract_detail( + _checkov_rule( + [ + { + "description": "Ensure S3 buckets are encrypted", + "keys": ["aws_s3_bucket.data.rule.sse_algorithm", "aws_s3_bucket.data.resource_type"], + }, + ] + ) + ) + + assert resources == ["aws_s3_bucket.data"] + + +def test_a_checkov_finding_with_no_keys_still_reports_its_description(): + messages, resources = render._extract_detail(_checkov_rule([{"description": "Some check", "keys": []}])) + + assert messages == ["Some check"] + assert resources == [] + + +@pytest.mark.parametrize("key", ["", "single", None, 42]) +def test_a_malformed_key_is_skipped_rather_than_crashing(key): + _messages, resources = render._extract_detail(_checkov_rule([{"description": "x", "keys": [key]}])) + + assert resources == [] + + +def test_the_tirith_shape_still_renders(): + """Teaching the renderer Checkov must not cost it the shape it already understood.""" + messages, resources = render._extract_detail( + { + "evaluations": { + "fails": [ + {"result": [{"message": "`3` is not equal to `0`", "meta": {"address": "null_resource.untagged"}}]} + ] + } + } + ) + + assert messages == ["`3` is not equal to `0`"] + assert resources == ["null_resource.untagged"] + + +def test_an_empty_description_does_not_hide_the_finding(): + """ + The exact shape a tirith rule with no declared description produces, taken from a QA run of the + cost policy `DO_NOT_TOUCH / cost-control`: + + {"id": ..., "description": "", "result": [{"message": "`23.832` is not less than `20`", ...}]} + + Both keys are present. Dispatching on `"description" in entry` took the Checkov path, found an + empty string to report, and skipped `result` -- so the policy appeared in the summary table with + an empty
block. A reviewer saw that a cost rule had tripped and no reason why. + """ + messages, resources = render._extract_detail( + { + "evaluations": { + "fails": [ + { + "id": "max-price-monthly-20", + "description": "", + "result": [{"passed": False, "message": "`23.832` is not less than `20`", "meta": None}], + "passed": False, + } + ] + } + } + ) + + assert messages == ["`23.832` is not less than `20`"] + # meta is None on the infracost provider -- only terraform_plan populates an address. + assert resources == [] + + +def test_an_entry_carrying_both_shapes_reports_both(): + """Reading both is additive, so neither shape can mask the other.""" + messages, resources = render._extract_detail( + { + "evaluations": { + "fails": [ + { + "description": "Ensure RDS is encrypted at rest", + "keys": ["aws_db_instance.db.storage_encrypted"], + "result": [ + {"message": "`false` is not equal to `true`", "meta": {"address": "aws_db_instance.db"}} + ], + } + ] + } + } + ) + + assert messages == ["Ensure RDS is encrypted at rest", "`false` is not equal to `true`"] + assert resources == ["aws_db_instance.db"] + + +def test_an_engine_error_is_still_surfaced_verbatim(): + messages, _resources = render._extract_detail( + {"evaluations": {"fails": [{"exec_err": "Checkov policy has no configPolicyIds"}]}} + ) + + assert messages == ["engine: Checkov policy has no configPolicyIds"] + + +# --- the scanned commit -------------------------------------------------------------------------- +# +# The comment is edited in place across runs, so without this a reader cannot tell whether the +# verdict in front of them is about the head of the branch or about a push from an hour ago. + + +def test_the_scanned_commit_is_rendered_under_the_headline(): + body = render.render_markdown(_results(), "COMPLETED", "https://run", commit="9ea6388f1c2d3e4f5a6b") + + lines = body.split("\n") + heading = next(i for i, line in enumerate(lines) if line.startswith("## ")) + assert lines[heading + 2] == "Scanned commit 9ea6388", lines[: heading + 4] + + +def test_no_commit_line_when_none_is_supplied(): + body = render.render_markdown(_results(), "COMPLETED", "https://run") + + assert "Scanned commit" not in body + + +def test_the_commit_line_survives_alongside_the_marker(): + """The marker has to stay line 1 -- it is what finds the comment again.""" + marker = "[//]: <> (tirith-comment, tag=default)" + body = render.render_markdown(_results(), "COMPLETED", "https://run", marker=marker, commit="abc1234def") + + assert body.startswith(marker) + assert "abc1234" in body + + +def test_a_non_sha_revision_is_not_truncated(): + """A tag or branch name is more useful whole; truncating one invents something sha-shaped.""" + body = render.render_markdown(_results(), "COMPLETED", "https://run", commit="release-2026-08") + + assert "release-2026-08" in body + + +def test_a_short_sha_is_left_alone(): + body = render.render_markdown(_results(), "COMPLETED", "https://run", commit="abc1234") + + assert "abc1234" in body + + +# --- a paused run, and results this module cannot read -------------------------------------------- + + +def test_a_fail_is_never_downgraded_by_a_paused_run(): + """ + The regression this pins: the APPROVAL_REQUIRED branch returned before the FAIL check, so a + paused run carrying a failing policy reported `warned` -- a neutral check, which SATISFIES a + required status check -- while the headline on the same counts said "1 failed". + """ + counts = {"FAIL": 1, "PASS": 2} + + assert render.verdict(counts, "APPROVAL_REQUIRED") == "failed" + + +def test_a_rule_with_no_result_is_not_a_pass(): + """`rule.get("result", PASS)` turned "the step wrote no verdict" into a clean bill of health.""" + counts, findings = render.summarize({"p": [{"rule_name": "r"}]}) + + assert counts[render.UNKNOWN] == 1 + assert counts[render.PASS] == 0 + assert render.verdict(counts, "COMPLETED") == "errored" + assert findings[0]["result"] == render.UNKNOWN + + +def test_a_result_this_module_does_not_recognise_is_not_silently_dropped(): + """ + An unrecognised value used to land in a count key `verdict` never inspects, so it vanished: the + run reported `no-policies` and exited 0. + """ + counts, _ = render.summarize({"p": [{"rule_name": "r", "result": "ERROR"}]}) + + assert render.verdict(counts, "COMPLETED") == "errored" + + +def test_a_fail_still_outranks_an_unreadable_result(): + counts, _ = render.summarize({"p": [{"rule_name": "a", "result": "FAIL"}, {"rule_name": "b", "result": "?"}]}) + + assert render.verdict(counts, "COMPLETED") == "failed" + + +# --- hostile input: the report must not be spoofable (pentest F1) -------------------------------- +# +# A pull-request author controls the terraform a plan is built from, so evaluator messages, resource +# addresses, rule names and policy ids are all attacker-influenced. Before this, every one of them was +# interpolated raw or wrapped in a single backtick -- and a backtick in the value closes that span, so +# the rest rendered as markdown and HTML. A pen test used it to put a fake "all policies passed" banner +# and a link whose text said app.stackguardian.io and whose href said somewhere else into the comment a +# reviewer reads. The gate itself was never affected; the report was. +# +# These assert against markdown RENDERED by a CommonMark parser, not against the source. The payload is +# still present in the source by design -- inside a code span, where it is inert -- so a substring check +# on the source proves nothing. Getting that wrong is easy: it is the mistake made while writing these. + +PAYLOAD = ( + "`x` is not equal to `y``

All policies passed

" + "[app.stackguardian.io](https://evil.example) | broken | cell" +) + + +def _render(**overrides): + finding = { + "rule_name": "cost-control", + "result": "FAIL", + "evaluations": { + "fails": [{"result": [{"message": "ordinary message", "meta": {"address": "aws_s3_bucket.b"}}]}] + }, + } + policy_id = overrides.pop("policy_id", "DO_NOT_TOUCH") + if "message" in overrides: + finding["evaluations"]["fails"][0]["result"][0]["message"] = overrides.pop("message") + if "address" in overrides: + finding["evaluations"]["fails"][0]["result"][0]["meta"]["address"] = overrides.pop("address") + finding.update(overrides) + return render.render_markdown({policy_id: [finding]}, "COMPLETED", "https://dash.example/run/1") + + +def _html_of(body): + """Render as GitHub would, so the assertions are about what a reviewer's browser receives.""" + pytest.importorskip("markdown_it", reason="needs markdown-it-py to render the assertion subject") + from markdown_it import MarkdownIt + + return MarkdownIt("commonmark").enable("table").render(body) + + +@pytest.mark.parametrize("field", ["message", "address", "rule_name", "policy_id"]) +def test_no_field_can_inject_markup_into_the_report(field): + """ + Every attacker-influenced field, through the same payload. `rule_name` mattered most: it was the one + field interpolated with no wrapping at all, straight into the `` element. + """ + rendered = _html_of(_render(**{field: PAYLOAD})) + + assert "

" not in rendered, f"{field} injected a heading" + assert 'href="https://evil.example"' not in rendered, f"{field} injected a link" + assert rendered.count("
") == rendered.count("
"), f"{field} broke the collapsible" + + +def test_the_payload_is_still_readable_after_being_neutralised(): + """ + Neutralising must not mean hiding. A reviewer has to be able to see what the policy actually + compared, or the fix trades a spoofing bug for a blind gate. + """ + rendered = _html_of(_render(message=PAYLOAD)) + + assert "All policies passed" in rendered + assert "<h1>" in rendered, "the markup should be shown as text, not dropped" + + +def test_a_pipe_or_newline_in_a_table_cell_keeps_the_row_intact(): + """ + A pipe splits a cell and a newline ends the row, so either one silently drops the real columns. + GFM's remedy is a backslash escape, which is the one escape that works inside a code span. + """ + body = _render(rule_name="a | b\nsecond line", address="x | y") + + rows = [line for line in body.splitlines() if line.startswith("|")] + assert len(rows) == 3, f"expected header, separator and one row; got {len(rows)}" + # Count only *unescaped* pipes -- an escaped `\|` is content, which is the whole point. + separators = len(re.findall(r"(?" in line) + assert "`" not in summary, f"an unescaped backtick survived into the summary: {summary}" + assert "`" in summary, "the backtick should be shown as an entity, not dropped" + + +def test_the_run_url_cannot_break_out_of_the_href(): + from html.parser import HTMLParser + + class Anchors(HTMLParser): + def __init__(self): + super().__init__() + self.attrs_seen = [] + + def handle_starttag(self, tag, attrs): + if tag == "a": + self.attrs_seen.append(dict(attrs)) + + body = render.render_markdown({}, "COMPLETED", 'https://dash.example/1" onmouseover="alert(1)') + parser = Anchors() + parser.feed(_html_of(body)) + + assert parser.attrs_seen, "expected the run link to be rendered" + for attributes in parser.attrs_seen: + assert list(attributes) == ["href"], f"the URL introduced an attribute: {list(attributes)}" + + +def test_a_benign_value_renders_exactly_as_before(): + """ + The regression guard for the fence approach: with no backticks in the value the fence is a single + backtick, so ordinary reports are byte-identical to what they were. If this breaks, every report + changed appearance and the diff is bigger than intended. + """ + body = _render() + + assert "| ❌ | `DO_NOT_TOUCH` | `cost-control` | `aws_s3_bucket.b` |" in body + assert "- `ordinary message`" in body diff --git a/tests/providers/json/policy_mixed_queries.json b/tests/providers/json/policy_mixed_queries.json new file mode 100644 index 00000000..e28679a8 --- /dev/null +++ b/tests/providers/json/policy_mixed_queries.json @@ -0,0 +1,131 @@ +{ + "meta": { + "version": "v1", + "required_provider": "stackguardian/json", + "policy_name": "Mixed Query Language Example", + "policy_description": "Demonstrates using both JMESPath and jq_query in the same policy" + }, + "evaluators": [ + { + "id": "jmespath_check_region", + "description": "Use JMESPath for simple field extraction", + "provider_args": { + "operation_type": "jmespath", + "query": "[0].vars.region" + }, + "condition": { + "type": "Equals", + "value": "us-east-1" + } + }, + { + "id": "jq_query_check_become", + "description": "Use jq_query for boolean checks", + "provider_args": { + "operation_type": "jq_query", + "query": ".[0].become" + }, + "condition": { + "type": "Equals", + "value": true + } + }, + { + "id": "jmespath_task_count", + "description": "Use JMESPath length function", + "provider_args": { + "operation_type": "jmespath", + "query": "length([0].tasks)" + }, + "condition": { + "type": "GreaterThan", + "value": 5 + } + }, + { + "id": "jq_query_filter_service_tasks", + "description": "Use jq_query for complex filtering", + "provider_args": { + "operation_type": "jq_query", + "query": "[.[0].tasks[] | select(has(\"service\"))] | length" + }, + "condition": { + "type": "GreaterThan", + "value": 0 + } + }, + { + "id": "jmespath_contains_check", + "description": "Use JMESPath contains for array membership", + "provider_args": { + "operation_type": "jmespath", + "query": "[0].tasks[*].name" + }, + "condition": { + "type": "Contains", + "value": "Start MySQL service" + } + }, + { + "id": "jq_query_conditional_logic", + "description": "Use jq_query for conditional transformations", + "provider_args": { + "operation_type": "jq_query", + "query": "if .[0].become then \"privileged\" else \"unprivileged\" end" + }, + "condition": { + "type": "Equals", + "value": "privileged" + } + }, + { + "id": "jmespath_projection", + "description": "Use JMESPath for multi-select projection", + "provider_args": { + "operation_type": "jmespath", + "query": "[0].{playbook_name: name, host_group: hosts}" + }, + "condition": { + "type": "RegexMatch", + "value": ".*Configure MySQL.*" + } + }, + { + "id": "jq_query_type_validation", + "description": "Use jq_query for type checking", + "provider_args": { + "operation_type": "jq_query", + "query": ".[0].tasks | type" + }, + "condition": { + "type": "Equals", + "value": "array" + } + }, + { + "id": "get_value_simple", + "description": "Use classic get_value for straightforward paths", + "provider_args": { + "operation_type": "get_value", + "key_path": "[0].hosts" + }, + "condition": { + "type": "Equals", + "value": "mysql_servers" + } + }, + { + "id": "jq_query_map_transform", + "description": "Use jq_query map for array transformations", + "provider_args": { + "operation_type": "jq_query", + "query": "[.[0].tasks[] | select(has(\"mysql\")) | .name]" + }, + "condition": { + "type": "Contains", + "value": "Create application database" + } + } + ], + "eval_expression": "(jmespath_check_region && jq_query_check_become) && (jmespath_task_count || jq_query_filter_service_tasks) && jmespath_contains_check && get_value_simple" +} diff --git a/tests/test_readme_is_current.py b/tests/test_readme_is_current.py new file mode 100644 index 00000000..7a02da86 --- /dev/null +++ b/tests/test_readme_is_current.py @@ -0,0 +1,132 @@ +""" +The README's generated bits must match what the program actually prints. + +Three things in it were hand-copied and had gone stale: the `## Usage` block was a paste of an older +`--help` missing `-var-path`, `-var` and the whole `platform` subcommand; the install-verification step +showed `1.0.0-beta.12` against a shipped `1.2.0`; and the Getting Started sample output predated the +current message format, so the first command a new user runs printed something different from the +documentation. + +Correcting the text is a one-off; it had already been corrected before and rotted again. What stops +that is checking it, so these run in CI. They compare against the real program output rather than +against a golden file, so adding a flag updates the requirement automatically -- the README is what has +to move. +""" + +import os +import re +import subprocess +import sys + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SRC = os.path.join(ROOT, "src") +README = os.path.join(ROOT, "README.md") + +sys.path.insert(0, SRC) + +from tirith import __version__ + + +def _readme(): + with open(README) as f: + return f.read() + + +def _help(*args): + """Run the CLI's --help the way a user would, in a subprocess, not by calling into argparse.""" + argv = list(args) + ["--help"] + code = ( + "import sys\n" + f"sys.argv = ['tirith'] + {argv!r}\n" + "from tirith.cli import main\n" + "try:\n" + " main()\n" + "except SystemExit:\n" + " pass\n" + ) + env = dict(os.environ, PYTHONPATH=SRC) + return subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, env=env).stdout + + +def _fenced_block_after(heading): + text = _readme() + start = text.index(heading) + len(heading) + open_fence = text.index("```", start) + close_fence = text.index("```", open_fence + 3) + return text[open_fence + 3 : close_fence].strip("\n") + + +def _options(text): + """Every option string in a help text, e.g. `{-policy-path, --json, --fail-on-error}`.""" + return set(re.findall(r"(?