diff --git a/.github/SBOM-README.md b/.github/SBOM-README.md new file mode 100644 index 000000000..123c75629 --- /dev/null +++ b/.github/SBOM-README.md @@ -0,0 +1,58 @@ +# SBOM & Vulnerability Scanning Automation + +This repository uses GitHub Actions to automatically generate a Software Bill of Materials (SBOM), scan for vulnerabilities, and produce package inventory reports. + +All reports are named with the repository name for easy identification. + +## Features + +SBOM Generation: Uses Syft to generate an SPDX JSON SBOM. +SBOM Merging: Merges SBOMs for multiple tools if needed. +SBOM to CSV: Converts SBOM JSON to a CSV report. +Vulnerability Scanning: Uses Grype to scan the SBOM for vulnerabilities and outputs a CSV report. +Package Inventory: Extracts a simple package list (name, type, version) as a CSV. +Artifacts: All reports are uploaded as workflow artifacts with the repository name in the filename. + +## Workflow Overview + +The main workflow is defined in .github/workflows/sbom.yml + +## Scripts + +scripts/create-sbom.sh +Generates an SBOM for the repo and for specified tools, merging them as needed. +scripts/update-sbom.py +Merges additional SBOMs into the main SBOM. +.github/scripts/sbom_json_to_csv.py +Converts the SBOM JSON to a detailed CSV report. +.github/scripts/grype_json_to_csv.py +Converts Grype’s vulnerability scan JSON output to a CSV report. +Output columns: REPO, NAME, INSTALLED, FIXED-IN, TYPE, VULNERABILITY, SEVERITY +.github/scripts/sbom_packages_to_csv.py +Extracts a simple package inventory from the SBOM. +Output columns: name, type, version + +## Example Reports + +Vulnerability Report +grype-report-[RepoName].csv +REPO,NAME,INSTALLED,FIXED-IN,TYPE,VULNERABILITY,SEVERITY +my-repo,Flask,2.1.2,,library,CVE-2022-12345,High +... + +Package Inventory +sbom-packages-[RepoName].csv +name,type,version +Flask,library,2.1.2 +Jinja2,library,3.1.2 +... + +## Usage + +Push to main branch or run the workflow manually. +Download artifacts from the workflow run summary. + +## Customization + +Add more tools to scripts/create-sbom.sh as needed. +Modify scripts to adjust report formats or add more metadata. diff --git a/.github/scripts/grype_json_to_csv.py b/.github/scripts/grype_json_to_csv.py new file mode 100644 index 000000000..622545505 --- /dev/null +++ b/.github/scripts/grype_json_to_csv.py @@ -0,0 +1,28 @@ +import json +import csv +import sys + +input_file = sys.argv[1] if len(sys.argv) > 1 else "grype-report.json" +output_file = sys.argv[2] if len(sys.argv) > 2 else "grype-report.csv" + +with open(input_file, "r", encoding="utf-8") as f: + data = json.load(f) + +columns = ["NAME", "INSTALLED", "FIXED-IN", "TYPE", "VULNERABILITY", "SEVERITY"] + +with open(output_file, "w", newline="", encoding="utf-8") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=columns) + writer.writeheader() + for match in data.get("matches", []): + pkg = match.get("artifact", {}) + vuln = match.get("vulnerability", {}) + row = { + "NAME": pkg.get("name", ""), + "INSTALLED": pkg.get("version", ""), + "FIXED-IN": vuln.get("fix", {}).get("versions", [""])[0] if vuln.get("fix", {}).get("versions") else "", + "TYPE": pkg.get("type", ""), + "VULNERABILITY": vuln.get("id", ""), + "SEVERITY": vuln.get("severity", ""), + } + writer.writerow(row) +print(f"CSV export complete: {output_file}") diff --git a/.github/scripts/sbom_json_to_csv.py b/.github/scripts/sbom_json_to_csv.py new file mode 100644 index 000000000..b11dfea9f --- /dev/null +++ b/.github/scripts/sbom_json_to_csv.py @@ -0,0 +1,78 @@ +import json +import csv +import sys +# from pathlib import Path +from tabulate import tabulate + +input_file = sys.argv[1] if len(sys.argv) > 1 else "sbom.json" +output_file = sys.argv[2] if len(sys.argv) > 2 else "sbom.csv" + +with open(input_file, "r", encoding="utf-8") as f: + sbom = json.load(f) + +packages = sbom.get("packages", []) + +columns = [ + "name", + "versionInfo", + "type", + "supplier", + "downloadLocation", + "licenseConcluded", + "licenseDeclared", + "externalRefs" +] + + +def get_type(pkg): + spdxid = pkg.get("SPDXID", "") + if "-" in spdxid: + parts = spdxid.split("-") + if len(parts) > 2: + return parts[2] + refs = pkg.get("externalRefs", []) + for ref in refs: + if ref.get("referenceType") == "purl": + return ref.get("referenceLocator", "").split("/")[0] + return "" + + +def get_external_refs(pkg): + refs = pkg.get("externalRefs", []) + return ";".join([ref.get("referenceLocator", "") for ref in refs]) + + +with open(output_file, "w", newline="", encoding="utf-8") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=columns) + writer.writeheader() + for pkg in packages: + row = { + "name": pkg.get("name", ""), + "versionInfo": pkg.get("versionInfo", ""), + "type": get_type(pkg), + "supplier": pkg.get("supplier", ""), + "downloadLocation": pkg.get("downloadLocation", ""), + "licenseConcluded": pkg.get("licenseConcluded", ""), + "licenseDeclared": pkg.get("licenseDeclared", ""), + "externalRefs": get_external_refs(pkg) + } + writer.writerow(row) + +print(f"CSV export complete: {output_file}") + + +with open("sbom_table.txt", "w", encoding="utf-8") as f: + table = [] + for pkg in packages: + row = [ + pkg.get("name", ""), + pkg.get("versionInfo", ""), + get_type(pkg), + pkg.get("supplier", ""), + pkg.get("downloadLocation", ""), + pkg.get("licenseConcluded", ""), + pkg.get("licenseDeclared", ""), + get_external_refs(pkg) + ] + table.append(row) + f.write(tabulate(table, columns, tablefmt="grid")) diff --git a/.github/scripts/sbom_packages_to_csv.py b/.github/scripts/sbom_packages_to_csv.py new file mode 100644 index 000000000..a7df2f4be --- /dev/null +++ b/.github/scripts/sbom_packages_to_csv.py @@ -0,0 +1,28 @@ +import json +import csv +import sys +import os + +input_file = sys.argv[1] if len(sys.argv) > 1 else "sbom.json" +repo_name = sys.argv[2] if len(sys.argv) > 2 else os.getenv("GITHUB_REPOSITORY", "unknown-repo").split("/")[-1] +output_file = f"sbom-packages-{repo_name}.csv" + +with open(input_file, "r", encoding="utf-8") as f: + sbom = json.load(f) + +packages = sbom.get("packages", []) + +columns = ["name", "type", "version"] + +with open(output_file, "w", newline="", encoding="utf-8") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=columns) + writer.writeheader() + for pkg in packages: + row = { + "name": pkg.get("name", ""), + "type": pkg.get("type", ""), + "version": pkg.get("versionInfo", "") + } + writer.writerow(row) + +print(f"Package list CSV generated: {output_file}") diff --git a/.github/workflows/sbom.yml b/.github/workflows/sbom.yml new file mode 100644 index 000000000..7b57a5db7 --- /dev/null +++ b/.github/workflows/sbom.yml @@ -0,0 +1,110 @@ +name: SBOM Vulnerability Scanning + +on: + workflow_dispatch: + inputs: + environment: + description: "Run SBOM check" + required: true + type: choice + options: + - yes + - no + +env: + SYFT_VERSION: "1.27.1" + TF_VERSION: "1.12.2" + +jobs: + deploy: + name: Software Bill of Materials + runs-on: ubuntu-latest + permissions: + actions: read + contents: write + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Setup Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Setup Terraform + uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd + + - uses: terraform-linters/setup-tflint@ae78205cfffec9e8d93fd2b3115c7e9d3166d4b6 + name: Setup TFLint + + - name: Set architecture variable + id: os-arch + run: | + case "${{ runner.arch }}" in + X64) ARCH="amd64" ;; + ARM64) ARCH="arm64" ;; + esac + echo "arch=${ARCH}" >> $GITHUB_OUTPUT + + - name: Download and setup Syft + run: | + DOWNLOAD_URL="https://github.com/anchore/syft/releases/download/v${{ env.SYFT_VERSION }}/syft_${{ env.SYFT_VERSION }}_linux_${{ steps.os-arch.outputs.arch }}.tar.gz" + echo "Downloading: ${DOWNLOAD_URL}" + + curl -L -o syft.tar.gz "${DOWNLOAD_URL}" + tar -xzf syft.tar.gz + chmod +x syft + + # Add to PATH for subsequent steps + echo "$(pwd)" >> $GITHUB_PATH + + - name: Create SBOM + run: bash scripts/create-sbom.sh terraform python tflint + + - name: Convert SBOM JSON to CSV + run: | + pip install --upgrade pip + pip install tabulate + REPO_NAME=$(basename $GITHUB_REPOSITORY) + python .github/scripts/sbom_json_to_csv.py sbom.json SBOM_${REPO_NAME}.csv + + - name: Upload SBOM CSV as artifact + uses: actions/upload-artifact@v4 + with: + name: sbom-csv + path: SBOM_${{ github.event.repository.name }}.csv + + - name: Install Grype + run: | + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin + + - name: Scan SBOM for Vulnerabilities (JSON) + run: | + grype sbom:sbom.json -o json > grype-report.json + + + + - name: Convert Grype JSON to CSV + run: | + pip install --upgrade pip + REPO_NAME=$(basename $GITHUB_REPOSITORY) + python .github/scripts/grype_json_to_csv.py grype-report.json grype-report-${REPO_NAME}.csv + + + - name: Upload Vulnerability Report + uses: actions/upload-artifact@v4 + with: + name: grype-report + path: grype-report-${{ github.event.repository.name }}.csv + + - name: Generate Package Inventory CSV + run: | + pip install --upgrade pip + REPO_NAME=$(basename $GITHUB_REPOSITORY) + python .github/scripts/sbom_packages_to_csv.py sbom.json $REPO_NAME + + - name: Upload Package Inventory CSV + uses: actions/upload-artifact@v4 + with: + name: sbom-packages + path: sbom-packages-${{ github.event.repository.name }}.csv \ No newline at end of file diff --git a/ansible/roles/build-ecs-proxies/tasks/build-container.yml b/ansible/roles/build-ecs-proxies/tasks/build-container.yml index 69cd89c4e..2554dc49e 100644 --- a/ansible/roles/build-ecs-proxies/tasks/build-container.yml +++ b/ansible/roles/build-ecs-proxies/tasks/build-container.yml @@ -10,22 +10,24 @@ changed_when: no - name: pull latest if exists - to ensure cache {{ item }} - docker_image: - name: "{{ ecr_registry }}/{{ item }}:{{ describe_images.stdout }}" - source: pull + ansible.builtin.command: + cmd: "docker pull {{ ecr_registry }}/{{ item }}:{{ describe_images.stdout }}" when: describe_images.stdout != 'None' ignore_errors: true no_log: true -- name: build and push ecr image {{ item }} - docker_image: - name: "{{ image_name }}" - build: - path: "{{ base_dir }}/{{ images_map[item].path }}" - dockerfile: "{{ base_dir }}/{{ images_map[item].dockerfile }}" - pull: yes - network: host - source: build - force_source: yes - push: yes - timeout: 300 +- name: build ecr image {{ item }} + ansible.builtin.command: + cmd: > + docker build + -t {{ image_name }} + -f {{ base_dir }}/{{ images_map[item].dockerfile }} + {{ base_dir }}/{{ images_map[item].path }} + --network host --pull + register: build_result + ignore_errors: false + +- name: push ecr image {{ item }} + ansible.builtin.command: + cmd: "docker push {{ image_name }}" + when: build_result.rc == 0 diff --git a/azure/common/deploy-stage.yml b/azure/common/deploy-stage.yml index c89ed35f1..d301afe99 100644 --- a/azure/common/deploy-stage.yml +++ b/azure/common/deploy-stage.yml @@ -241,7 +241,7 @@ stages: fi displayName: Check supplied names against API registry continueOnError: true - + - template: '../templates/deploy-service.yml' parameters: service_name: ${{ parameters.service_name }} diff --git a/azure/templates/deploy-service.yml b/azure/templates/deploy-service.yml index a67b52457..0a49c69d3 100644 --- a/azure/templates/deploy-service.yml +++ b/azure/templates/deploy-service.yml @@ -249,7 +249,7 @@ steps: export PULL_REQUEST="${{ parameters.pr_label }}" make --no-print-directory -C $(UTILS_DIR)/ansible deploy-manifest displayName: 'Deploy Manifest' - + - ${{ if parameters.proxy_path }}: - bash: | set -euo pipefail diff --git a/scripts/copy_spec.py b/scripts/copy_spec.py new file mode 100644 index 000000000..bef79b3b2 --- /dev/null +++ b/scripts/copy_spec.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +import os +import sys +import boto3 +import json +from pathlib import Path +from botocore.exceptions import ClientError + +def upload_to_s3(file_path: Path, bucket_name: str, folder_name: str): + """Upload one file to the given S3 bucket under folder_name/.""" + s3 = boto3.client("s3") + key = f"apis/{folder_name}/{file_path}" + + try: + s3.upload_file(str(file_path), bucket_name, key) + print(f"[OK] Uploaded → s3://{bucket_name}/apis/{key}") + except ClientError as e: + print(f"[ERROR] Upload failed: {file_path} → s3://{bucket_name}/{key}") + print(e) + sys.exit(1) + + +def main(bucket_name: str, repo_name: str): + cwd = os.getcwd() + print("Current working directory:", cwd) + + root_dir = Path.cwd().parents[1] + json_file = root_dir / f"{repo_name}.json" + minified_json = "spec.json" + + print(json_file) + + if not json_file.is_file(): + print(f"[ERROR] JSON spec not found: {json_file}") + return 1 + + with open(json_file, "r") as f: + data = json.load(f) + + with open(minified_json, "w") as f: + json.dump(data, f, separators=(",", ":")) + + upload_to_s3(minified_json, bucket_name, repo_name) + + print("[DONE] Processing complete.") + return 0 + +if __name__ == "__main__": + print("Hitting main") + + if len(sys.argv) != 3: + print("Usage: python copy_spec_to_s3.py ") + sys.exit(1) + + bucket_name = sys.argv[1] + repo_name = sys.argv[2] + print(f"Repo name: {repo_name}") + print(f"Bucket name: {bucket_name}") + + sys.exit(main(bucket_name,repo_name)) \ No newline at end of file diff --git a/scripts/create-sbom.sh b/scripts/create-sbom.sh new file mode 100644 index 000000000..44ff0311e --- /dev/null +++ b/scripts/create-sbom.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n\t' + +REPO_ROOT=$(git rev-parse --show-toplevel) + +# Generate SBOM for current directory +syft -o spdx-json . > "$REPO_ROOT/sbom.json" + +# Generate and merge SBOMs for each tool passed as argument +for tool in "$@"; do + echo "Creating SBOM for $tool and merging" + tool_path=$(command -v "$tool") + if [[ -z "$tool_path" ]]; then + echo "Warning: '$tool' not found in PATH. Skipping." >&2 + continue + fi + syft -q -o spdx-json "$tool_path" | python "$REPO_ROOT/scripts/update-sbom.py" +done \ No newline at end of file diff --git a/scripts/update-sbom.py b/scripts/update-sbom.py new file mode 100644 index 000000000..d804485ca --- /dev/null +++ b/scripts/update-sbom.py @@ -0,0 +1,21 @@ +import json +import sys +from pathlib import Path + + +def main() -> None: + with Path("sbom.json").open("r") as f: + sbom = json.load(f) + + tool = json.loads(sys.stdin.read()) + + sbom.setdefault("packages", []).extend(tool.setdefault("packages", [])) + sbom.setdefault("files", []).extend(tool.setdefault("files", [])) + sbom.setdefault("relationships", []).extend(tool.setdefault("relationships", [])) + + with Path("sbom.json").open("w") as f: + json.dump(sbom, f) + + +if __name__ == "__main__": + main() \ No newline at end of file