diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 2b327d8e9..7f4d927f3 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -34,7 +34,7 @@ jobs: - name: Find tests not covered by any named group id: find-uncovered run: | - # Combined regex of every pattern used across the 13 named matrix groups. + # Combined regex of every pattern used across the 14 named matrix groups. # Any test whose name does NOT match this will land in the catch-all group. # Built via concatenation so every line stays at ≥10-space YAML indentation. CP="TestCreateScan|TestScanCreate|TestScansE2E|TestFastScan" @@ -57,6 +57,7 @@ jobs: CP="${CP}|TestGetLearnMore|TestImport|TestGetTenant|TestMaskSecrets|TestFailedMask" CP="${CP}|TestScaRemediation|TestKicsRemediation|TestTelemetry|Test_Handle|TestChat" CP="${CP}|TestIntegrationScaResolver" + CP="${CP}|TestExcludeGitFolder" COVERED_PATTERNS="${CP}" ALL_TESTS=$(grep -rh "^func Test" test/integration/*_test.go \ @@ -85,7 +86,7 @@ jobs: fi # ───────────────────────────────────────────────────────────────────────────── - # Job B: Run each test group in parallel across 14 matrix entries. + # Job B: Run each test group in parallel across 15 matrix entries. # The 14th entry (uncovered) is a dynamic catch-all driven by Job A. # ───────────────────────────────────────────────────────────────────────────── integration-tests: @@ -200,7 +201,15 @@ jobs: needs_precommit: "false" run_cleandata: "true" - # 14 ── Catch-All (dynamic; pattern injected at runtime from Job A output) + # 14 ── Git Folder Exclusion & Contributors CSV (integration tests for --exclude-git-folder flag with CSV/JSON generation) + - name: git-folder-exclude + label: "Git Folder Exclusion & CSV" + run_pattern: "TestExcludeGitFolder" + timeout: "30m" + needs_precommit: "false" + run_cleandata: "false" + + # 15 ── Catch-All (dynamic; pattern injected at runtime from Job A output) - name: uncovered label: "Catch-All (Uncovered)" run_pattern: "__UNCOVERED__" diff --git a/.golangci.yml b/.golangci.yml index 4d4091850..449c93cce 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,125 +1,127 @@ -# .golangci.yml - -version: "2" -run: - timeout: 10m - -linters: - enable: - - bodyclose - - depguard - - dogsled - - dupl - - errcheck - - funlen - - gochecknoinits - - goconst - - gocritic - - gocyclo - - ineffassign - - mnd # replacement for gomnd - - nakedret - - revive # replacement for golint - - rowserrcheck - - staticcheck - - unconvert - - unparam - - unused # covers deadcode/varcheck/structcheck - - whitespace - exclusions: - paths: - - test/testdata_etc - - internal/cache - - internal/renameio - - internal/robustio - rules: - - path: _test\.go - linters: - - mnd - settings: - depguard: - rules: - main: - list-mode: lax - allow: - - $gostd - - github.com/checkmarx/ast-cli/internal - - github.com/gookit/color - - github.com/CheckmarxDev/containers-resolver/pkg/containerResolver - - github.com/Checkmarx/manifest-parser/pkg/parser/models - - github.com/Checkmarx/manifest-parser/pkg/parser - - github.com/Checkmarx/secret-detection/pkg/hooks/pre-commit - - github.com/Checkmarx/secret-detection/pkg/hooks/pre-receive - - github.com/Checkmarx/gen-ai-prompts/prompts/sast_result_remediation - - github.com/spf13/viper - - github.com/checkmarx/2ms/v3/lib/reporting - - github.com/checkmarx/2ms/v3/lib/secrets - - github.com/checkmarx/2ms/v3/pkg - - github.com/Checkmarx/gen-ai-wrapper - - github.com/spf13/cobra - - github.com/pkg/errors - - github.com/google - - github.com/MakeNowJust/heredoc - - github.com/jsumners/go-getport - - github.com/stretchr/testify/assert - - github.com/gofrs/flock - - github.com/golang-jwt/jwt/v5 - - github.com/checkmarx/go-keyring - - github.com/Checkmarx/containers-images-extractor/pkg/imagesExtractor - - github.com/Checkmarx/containers-syft-packages-extractor/pkg/syftPackagesExtractor - - github.com/Checkmarx/containers-types/types - dupl: - threshold: 500 - funlen: - lines: 200 - statements: 100 - goconst: - min-len: 2 - min-occurrences: 2 - gocritic: - enabled-tags: - - diagnostic - - experimental - - opinionated - - performance - - style - disabled-checks: - - dupImport # https://github.com/go-critic/go-critic/issues/845 - - ifElseChain - - octalLiteral - - whyNoLint - - wrapperFunc - gocyclo: - min-complexity: 15 - mnd: - checks: - - argument - - case - - condition - - return - revive: - rules: - - name: exported - arguments: - - disableStutteringCheck - govet: - settings: - printf: - funcs: - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf - lll: - line-length: 185 - misspell: - locale: US - -formatters: - enable: - - gofmt - - goimports - settings: - goimports: - local-prefixes: - - github.com/golangci/golangci-lint +# .golangci.yml + +version: "2" +run: + timeout: 10m + +linters: + enable: + - bodyclose + - depguard + - dogsled + - dupl + - errcheck + - funlen + - gochecknoinits + - goconst + - gocritic + - gocyclo + - ineffassign + - mnd # replacement for gomnd + - nakedret + - revive # replacement for golint + - rowserrcheck + - staticcheck + - unconvert + - unparam + - unused # covers deadcode/varcheck/structcheck + - whitespace + exclusions: + paths: + - test/testdata_etc + - internal/cache + - internal/renameio + - internal/robustio + rules: + - path: _test\.go + linters: + - mnd + settings: + depguard: + rules: + main: + list-mode: lax + allow: + - $gostd + - github.com/checkmarx/ast-cli/internal + - github.com/gookit/color + - github.com/CheckmarxDev/containers-resolver/pkg/containerResolver + - github.com/Checkmarx/manifest-parser/pkg/parser/models + - github.com/Checkmarx/manifest-parser/pkg/parser + - github.com/Checkmarx/secret-detection/pkg/hooks/pre-commit + - github.com/Checkmarx/secret-detection/pkg/hooks/pre-receive + - github.com/Checkmarx/gen-ai-prompts/prompts/sast_result_remediation + - github.com/spf13/viper + - github.com/checkmarx/2ms/v3/lib/reporting + - github.com/checkmarx/2ms/v3/lib/secrets + - github.com/checkmarx/2ms/v3/pkg + - github.com/Checkmarx/gen-ai-wrapper + - github.com/spf13/cobra + - github.com/pkg/errors + - github.com/google + - github.com/MakeNowJust/heredoc + - github.com/jsumners/go-getport + - github.com/stretchr/testify/assert + - github.com/gofrs/flock + - github.com/golang-jwt/jwt/v5 + - github.com/checkmarx/go-keyring + - github.com/Checkmarx/containers-images-extractor/pkg/imagesExtractor + - github.com/Checkmarx/containers-types/types + - github.com/go-git/go-git/v5 + - github.com/go-git/go-git/v5/plumbing + - github.com/go-git/go-git/v5/plumbing/object + dupl: + threshold: 500 + funlen: + lines: 200 + statements: 100 + goconst: + min-len: 2 + min-occurrences: 2 + gocritic: + enabled-tags: + - diagnostic + - experimental + - opinionated + - performance + - style + disabled-checks: + - dupImport # https://github.com/go-critic/go-critic/issues/845 + - ifElseChain + - octalLiteral + - whyNoLint + - wrapperFunc + gocyclo: + min-complexity: 15 + mnd: + checks: + - argument + - case + - condition + - return + revive: + rules: + - name: exported + arguments: + - disableStutteringCheck + govet: + settings: + printf: + funcs: + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf + lll: + line-length: 185 + misspell: + locale: US + +formatters: + enable: + - gofmt + - goimports + settings: + goimports: + local-prefixes: + - github.com/golangci/golangci-lint diff --git a/go.mod b/go.mod index 49d7be7f6..9eec06f4f 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( ) require ( - cyphar.com/go-pathrs v0.2.4 // indirect + cyphar.com/go-pathrs v0.2.5 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20240914100643-eb91380d8434 // indirect github.com/Masterminds/semver v1.5.0 // indirect @@ -55,7 +55,7 @@ require ( github.com/godbus/dbus/v5 v5.2.2 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/jsonschema-go v0.4.3 // indirect - github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/knqyf263/go-rpmdb v0.1.1 // indirect github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/mitchellh/mapstructure v1.5.1-0.20220423092549-19e70c243037 // indirect @@ -86,7 +86,7 @@ require ( github.com/Masterminds/squirrel v1.5.4 // indirect github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect github.com/Microsoft/hcsshim v0.15.0-rc.3 // indirect - github.com/ProtonMail/go-crypto v1.4.0 // indirect + github.com/ProtonMail/go-crypto v1.4.1 // indirect github.com/acobaugh/osrelease v0.1.0 // indirect github.com/adrg/xdg v0.5.3 // indirect github.com/agext/levenshtein v1.2.3 // indirect @@ -119,7 +119,7 @@ require ( github.com/charmbracelet/x/ansi v0.11.6 // indirect github.com/charmbracelet/x/cellbuf v0.0.15 // indirect github.com/charmbracelet/x/term v0.2.2 // indirect - github.com/cloudflare/circl v1.6.3 // indirect + github.com/cloudflare/circl v1.6.5 // indirect github.com/containerd/cgroups/v3 v3.1.3 // indirect github.com/containerd/containerd v1.7.35 // indirect github.com/containerd/containerd/api v1.10.0 // indirect @@ -131,7 +131,7 @@ require ( github.com/containerd/platforms v1.0.0-rc.4 // indirect github.com/containerd/ttrpc v1.2.8 // indirect github.com/containerd/typeurl/v2 v2.3.0 // indirect - github.com/cyphar/filepath-securejoin v0.6.1 // indirect + github.com/cyphar/filepath-securejoin v0.7.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deitch/magic v0.0.0-20240306090643-c67ab88f10cb // indirect github.com/distribution/reference v0.6.0 // indirect @@ -158,8 +158,8 @@ require ( github.com/gitleaks/go-gitdiff v0.9.1 // indirect github.com/go-errors/errors v1.5.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.9.0 // indirect - github.com/go-git/go-git/v5 v5.19.2 // indirect + github.com/go-git/go-billy/v5 v5.9.1 // indirect + github.com/go-git/go-git/v5 v5.19.2 github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -199,7 +199,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953 // indirect - github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/kevinburke/ssh_config v1.6.0 // indirect github.com/klauspost/compress v1.18.6 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect @@ -258,7 +258,7 @@ require ( github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sirupsen/logrus v1.9.4 // indirect - github.com/skeema/knownhosts v1.3.1 // indirect + github.com/skeema/knownhosts v1.3.2 // indirect github.com/slack-go/slack v0.23.1 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spdx/gordf v0.0.0-20250128162952-000978ccd6fb // indirect diff --git a/go.sum b/go.sum index 33b6c499c..07a1cdf4a 100644 --- a/go.sum +++ b/go.sum @@ -45,8 +45,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cyphar.com/go-pathrs v0.2.4 h1:iD/mge36swa1UFKdINkr1Frkpp6wZsy3YYEildj9cLY= -cyphar.com/go-pathrs v0.2.4/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= +cyphar.com/go-pathrs v0.2.5 h1:SnX9FBvnoyn3lUs1dkMgZ52bAETpirNu3FTRh5HlRik= +cyphar.com/go-pathrs v0.2.5/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -110,8 +110,8 @@ github.com/Microsoft/hcsshim v0.15.0-rc.3/go.mod h1:VhDiwXgb8cEJxO9H57YL4NNIYqvZ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= -github.com/ProtonMail/go-crypto v1.4.0 h1:Zq/pbM3F5DFgJiMouxEdSVY44MVoQNEKp5d5QxIQceQ= -github.com/ProtonMail/go-crypto v1.4.0/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= +github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= +github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= github.com/acobaugh/osrelease v0.1.0 h1:Yb59HQDGGNhCj4suHaFQQfBps5wyoKLSSX/J/+UifRE= github.com/acobaugh/osrelease v0.1.0/go.mod h1:4bFEs0MtgHNHBrmHCt67gNisnabCRAlzdVasCEGHTWY= github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78= @@ -241,8 +241,8 @@ github.com/clipperhouse/displaywidth v0.10.0 h1:GhBG8WuerxjFQQYeuZAeVTuyxuX+Urai github.com/clipperhouse/displaywidth v0.10.0/go.mod h1:XqJajYsaiEwkxOj4bowCTMcT1SgvHo9flfF3jQasdbs= github.com/clipperhouse/uax29/v2 v2.6.0 h1:z0cDbUV+aPASdFb2/ndFnS9ts/WNXgTNNGFoKXuhpos= github.com/clipperhouse/uax29/v2 v2.6.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= -github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= -github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= +github.com/cloudflare/circl v1.6.5 h1:O64F26HEqNhznd/hrC5KZXVKYuKM2rx4deZDTc4ihQA= +github.com/cloudflare/circl v1.6.5/go.mod h1:h5LNyxAc5nTue9DS5jT+48en2PSDYt3zdGnz5OstK6c= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -283,8 +283,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= -github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= +github.com/cyphar/filepath-securejoin v0.7.0 h1:s0Y3ITPy6sQn5xt54DuYvTF8hu134ooYLUb58DX/HjE= +github.com/cyphar/filepath-securejoin v0.7.0/go.mod h1:ymLGms/u3BYaviIiuKFnUx8EkQEZeK6cInNoAPJA3o4= github.com/danieljoos/wincred v1.2.3 h1:v7dZC2x32Ut3nEfRH+vhoZGvN72+dQ/snVXo/vMFLdQ= github.com/danieljoos/wincred v1.2.3/go.mod h1:6qqX0WNrS4RzPZ1tnroDzq9kY3fu1KwE7MRLQK4X0bs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -390,8 +390,8 @@ github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8b github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA= -github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= +github.com/go-git/go-billy/v5 v5.9.1 h1:8U73XiOTfINdItHVa6z4Gv7ToObcZ6grkqQbLryLCdA= +github.com/go-git/go-billy/v5 v5.9.1/go.mod h1:ExsU+jcGwXTBOnyilvAnEM1wug1IxHr4yP2ZXsNRtV0= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY= @@ -654,16 +654,16 @@ github.com/jsumners/go-getport v1.0.0/go.mod h1:KpeJgwNSkpuXuoGhJ2Hgl5QJqWbLG1m0 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953 h1:WdAeg/imY2JFPc/9CST4bZ80nNJbiBFCAdSZCSgrS5Y= github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953/go.mod h1:6o+UrvuZWc4UTyBhQf0LGjW9Ld7qJxLz/OqvSOWWlEc= -github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= -github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY= +github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= -github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/knqyf263/go-rpmdb v0.1.1 h1:oh68mTCvp1XzxdU7EfafcWzzfstUZAEa3MW0IJye584= @@ -923,8 +923,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= -github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= -github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= +github.com/skeema/knownhosts v1.3.2 h1:EDL9mgf4NzwMXCTfaxSD/o/a5fxDw/xL9nkU28JjdBg= +github.com/skeema/knownhosts v1.3.2/go.mod h1:bEg3iQAuw+jyiw+484wwFJoKSLwcfd7fqRy+N0QTiow= github.com/slack-go/slack v0.23.1 h1:ZS5B96wxxYQRwvJ3/vJFtqtUZi3tXhsZCyT44Nv7M80= github.com/slack-go/slack v0.23.1/go.mod h1:H0yR/YBuRJ39RkE+JpV/d/oEsbanzTRowR82bCN0cEs= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= diff --git a/internal/commands/gitmetadata.go b/internal/commands/gitmetadata.go new file mode 100644 index 000000000..bb6bc5b01 --- /dev/null +++ b/internal/commands/gitmetadata.go @@ -0,0 +1,582 @@ +package commands + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "fmt" + "net/http" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/pkg/errors" + + "github.com/checkmarx/ast-cli/internal/logger" +) + +const ( + // CheckmarxFolderName is the folder where CLI-generated metadata files are stored. + CheckmarxFolderName = ".checkmarx" + // ContributorsFileName is the filename for the CSV with contributor information. + ContributorsFileName = "contributors.csv" + // MetadataFileName is the filename for the JSON metadata file. + MetadataFileName = "metadata.json" + + commitHistoryWindow = 90 * 24 * time.Hour + defaultRemoteName = "origin" + contributorsSizeLimit = 1 * 1024 * 1024 // repostore ignores contributors.csv above this size + privacyDetectionTimeout = 5 // seconds for privacy detection HTTP requests + generatedFilePerm = 0o644 + generatedDirPerm = 0o755 + csvFieldCount = 4 + urlSchemeParts = 2 // parts when splitting by "://" + pathParts = 2 // parts when splitting by "/" + sshSplitParts = 2 // parts when splitting SSH URL by "@" or ":" +) + +// contributorsMetadata mirrors repostore metadata structure; omits branchName per tech design. +type contributorsMetadata struct { + RepositoryURL string `json:"repositoryUrl"` + LastCommitHash string `json:"lastCommitHash"` + LastCommitDate string `json:"lastCommitDate"` + CommitsCount int `json:"commitsCount"` +} + +// GenerateAndWrite creates contributors.csv (private only) and metadata.json under .checkmarx/; errors should be logged but not fail scan. +func GenerateAndWrite(repoPath string, isPrivateRepo bool) error { + repo, err := gogit.PlainOpen(repoPath) + if err != nil { + // Fallback to system git if go-git fails (e.g., unsupported git extensions like worktreeConfig). + if strings.Contains(err.Error(), "does not support extension") { + logger.PrintfIfVerbose("go-git cannot open repository (unsupported git extension), falling back to system git") + return generateViaSystemGit(repoPath, isPrivateRepo) + } + return errors.Wrap(err, "could not open local git repository") + } + + headCommit, err := resolveHeadCommit(repo) + if err != nil { + return errors.Wrap(err, "could not resolve HEAD commit") + } + + since := time.Now().Add(-commitHistoryWindow) + commits, err := commitsSince(repo, since) + if err != nil { + // Fallback to system git for shallow clones or corrupted repos where go-git can't read objects + if strings.Contains(err.Error(), "object not found") { + logger.PrintfIfVerbose("go-git cannot read commit history (shallow clone or corrupted repo), falling back to system git") + return generateViaSystemGit(repoPath, isPrivateRepo) + } + return errors.Wrap(err, "could not read commit history") + } + + // Generate CSV only for private repos + var csvData []byte + if isPrivateRepo { + csvData, err = buildContributorsCSV(commits) + if err != nil { + return errors.Wrap(err, "could not build contributors.csv") + } + if len(csvData) > contributorsSizeLimit { + logger.PrintfIfVerbose("contributors.csv is %d bytes, over the 1MB size repostore accepts", len(csvData)) + } + } + + // Always generate metadata.json (all repos) + metadataData, err := buildMetadataJSON(remoteURL(repo), headCommit, len(commits)) + if err != nil { + return errors.Wrap(err, "could not build metadata.json") + } + + return writeGeneratedFilesConditional(repoPath, csvData, metadataData, isPrivateRepo) +} + +// resolveHeadCommit resolves HEAD to its full commit object, not just the hash. +func resolveHeadCommit(repo *gogit.Repository) (*object.Commit, error) { + head, err := repo.Head() + if err != nil { + return nil, err + } + return repo.CommitObject(head.Hash()) +} + +// commitsSince returns commits from last 90 days (by author date, not committer date) sorted newest-first. +func commitsSince(repo *gogit.Repository, since time.Time) ([]*object.Commit, error) { + iter, err := repo.Log(&gogit.LogOptions{}) + if err != nil { + return nil, err + } + defer iter.Close() + + var commits []*object.Commit + err = iter.ForEach(func(c *object.Commit) error { + if !c.Author.When.Before(since) { + commits = append(commits, c) + } + return nil + }) + if err != nil { + return nil, err + } + sort.Slice(commits, func(i, j int) bool { + return commits[i].Author.When.After(commits[j].Author.When) + }) + return commits, nil +} + +// dedupByEmail keeps most recent commit per unique email to keep contributors.csv small. +func dedupByEmail(commits []*object.Commit) []*object.Commit { + seen := make(map[string]bool, len(commits)) + deduped := make([]*object.Commit, 0, len(commits)) + for _, c := range commits { + email := strings.ToLower(strings.TrimSpace(c.Author.Email)) + if seen[email] { + continue + } + seen[email] = true + deduped = append(deduped, c) + } + return deduped +} + +// buildContributorsCSV writes one row per unique email: date, hash, email, username. No header row. +func buildContributorsCSV(commits []*object.Commit) ([]byte, error) { + var buf strings.Builder + writer := csv.NewWriter(&buf) + + for _, c := range dedupByEmail(commits) { + row := []string{ + c.Author.When.Format(time.RFC3339), + c.Hash.String(), + c.Author.Email, + c.Author.Name, + } + if err := writer.Write(row); err != nil { + return nil, err + } + } + + writer.Flush() + if err := writer.Error(); err != nil { + return nil, err + } + return []byte(buf.String()), nil +} + +func buildMetadataJSON(repositoryURL string, headCommit *object.Commit, commitsCount int) ([]byte, error) { + metadata := contributorsMetadata{ + RepositoryURL: repositoryURL, + LastCommitHash: headCommit.Hash.String(), + LastCommitDate: headCommit.Author.When.Format(time.RFC3339), + CommitsCount: commitsCount, + } + return json.MarshalIndent(metadata, "", " ") +} + +// remoteURL returns the "origin" remote's first URL, or "" if there is none (e.g. a bare local clone). +func remoteURL(repo *gogit.Repository) string { + remote, err := repo.Remote(defaultRemoteName) + if err != nil { + return "" + } + urls := remote.Config().URLs + if len(urls) == 0 { + return "" + } + return urls[0] +} + +// generateViaSystemGit extracts repo info via system git when go-git fails (e.g., unsupported extensions). +func generateViaSystemGit(repoPath string, isPrivateRepo bool) error { + remoteURL, err := gitCommand(repoPath, "config", "--get", "remote.origin.url") + if err != nil { + return errors.Wrap(err, "could not get remote URL via system git") + } + + headHash, err := gitCommand(repoPath, "rev-parse", "HEAD") + if err != nil { + return errors.Wrap(err, "could not get HEAD commit via system git") + } + + sinceCutoff := time.Now().Add(-commitHistoryWindow) + sinceStr := sinceCutoff.Format(time.RFC3339) + logOutput, err := gitCommand(repoPath, "log", "--since="+sinceStr, "--pretty=format:%aI%x1f%H%x1f%ae%x1f%an") + if err != nil { + return errors.Wrap(err, "could not get commit log via system git") + } + + commits := parseGitLogOutput(logOutput, sinceCutoff) + if len(commits) == 0 { + commits = []map[string]string{} // empty list for builds with no commits in 90 days + } + + // Extract the actual last commit date (newest commit is first in array) + // If no commits in 90-day window, get HEAD date directly (consistent with go-git path) + lastCommitDateStr := "" + if len(commits) > 0 { + lastCommitDateStr = commits[0]["date"] + } else { + // No commits in 90-day window, but HEAD still exists - get its date + headDate, err := gitCommand(repoPath, "log", "-1", "--format=%aI", "HEAD") + if err == nil && headDate != "" { + lastCommitDateStr = headDate + } + } + + // Generate CSV only for private repos + var csvData []byte + if isPrivateRepo { + csvData, err = buildContributorsCSV(convertToCoreCommits(commits)) + if err != nil { + return errors.Wrap(err, "could not build contributors.csv") + } + if len(csvData) > contributorsSizeLimit { + logger.PrintfIfVerbose("contributors.csv is %d bytes, over the 1MB size repostore accepts", len(csvData)) + } + } + + // Always generate metadata.json + metadataData, err := buildMetadataJSONFromSystem(remoteURL, headHash, len(commits), lastCommitDateStr) + if err != nil { + return errors.Wrap(err, "could not build metadata.json") + } + + return writeGeneratedFilesConditional(repoPath, csvData, metadataData, isPrivateRepo) +} + +// gitCommand runs a git command in the given repo directory and returns trimmed output. +func gitCommand(repoPath string, args ...string) (string, error) { + cmd := exec.Command("git", args...) + cmd.Dir = repoPath + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + if err := cmd.Run(); err != nil { + return "", errors.Wrapf(err, "git command failed: %s", out.String()) + } + return strings.TrimSpace(out.String()), nil +} + +// parseGitLogOutput parses git log output (date, hash, email, name) filtered by author date and returns newest-first. +func parseGitLogOutput(logOutput string, sinceCutoff time.Time) []map[string]string { + if logOutput == "" { + return nil + } + + lines := strings.Split(logOutput, "\n") + commits := make([]map[string]string, 0, len(lines)) + + for _, line := range lines { + if line == "" { + continue + } + parts := strings.Split(line, "\x1f") + if len(parts) != csvFieldCount { + continue + } + + // Filter by author date to correctly handle rebased/cherry-picked commits + authorDate, err := time.Parse(time.RFC3339, parts[0]) + if err != nil { + // Skip commits with unparseable dates (shouldn't happen with %aI format) + continue + } + if !sinceCutoff.IsZero() && authorDate.Before(sinceCutoff) { + // Skip commits older than the cutoff + continue + } + + commits = append(commits, map[string]string{ + "date": parts[0], + "hash": parts[1], + "email": parts[2], + "name": parts[3], + }) + } + return commits +} + +// convertToCoreCommits converts system-git commit maps to go-git Commit objects for CSV building. +func convertToCoreCommits(commits []map[string]string) []*object.Commit { + result := make([]*object.Commit, 0, len(commits)) + for _, c := range commits { + // Parse ISO8601 commit date; fall back to zero time on error. + commitTime, _ := time.Parse(time.RFC3339, c["date"]) + // Parse the commit hash; fall back to zero hash on error. + hash := plumbing.NewHash(c["hash"]) + commit := &object.Commit{ + Hash: hash, + Author: object.Signature{ + Email: c["email"], + Name: c["name"], + When: commitTime, + }, + } + result = append(result, commit) + } + return result +} + +// buildMetadataJSONFromSystem builds metadata.json using data from system git. +func buildMetadataJSONFromSystem(remoteURL, headHash string, commitCount int, lastCommitDate string) ([]byte, error) { + metadata := contributorsMetadata{ + RepositoryURL: remoteURL, + LastCommitHash: headHash, + LastCommitDate: lastCommitDate, + CommitsCount: commitCount, + } + return json.Marshal(metadata) +} + +// writeGeneratedFilesConditional always writes metadata.json; only writes contributors.csv for private repos. +func writeGeneratedFilesConditional(repoPath string, csvData, metadataData []byte, isPrivateRepo bool) error { + checkmarxDir := filepath.Join(repoPath, CheckmarxFolderName) + if err := os.MkdirAll(checkmarxDir, generatedDirPerm); err != nil { + return errors.Wrap(err, "could not create .checkmarx directory") + } + + // ALWAYS write metadata.json (all repos) + if err := os.WriteFile(filepath.Join(checkmarxDir, MetadataFileName), metadataData, generatedFilePerm); err != nil { + return errors.Wrap(err, "could not write "+MetadataFileName) + } + + // ONLY write contributors.csv for private repos + if isPrivateRepo { + if err := os.WriteFile(filepath.Join(checkmarxDir, ContributorsFileName), csvData, generatedFilePerm); err != nil { + return errors.Wrap(err, "could not write "+ContributorsFileName) + } + } + + return nil +} + +// detectRepositoryPrivacy determines if repo is private/public; conservatively defaults to private for unknown repos. +func detectRepositoryPrivacy(repoPath string, httpClient *http.Client) bool { + // Method 1: Try go-git + repo, err := gogit.PlainOpen(repoPath) + if err == nil { + return isPrivateByURL(remoteURL(repo), httpClient) + } + + // Method 2: Try system git + remoteURL, err := gitCommand(repoPath, "config", "--get", "remote.origin.url") + if err == nil && remoteURL != "" { + return isPrivateByURL(remoteURL, httpClient) + } + + // Method 3: Check if contributors.csv exists (was private) + checkmarxDir := filepath.Join(repoPath, CheckmarxFolderName) + csvPath := filepath.Join(checkmarxDir, ContributorsFileName) + if fileExists(csvPath) { + return true + } + + // Default: Conservative (treat as private) + return true +} + +// isPrivateByURL detects repository privacy via public APIs (GitHub/GitLab/Bitbucket/Azure); defaults to private on any error. +func isPrivateByURL(remoteURL string, httpClient *http.Client) bool { + if remoteURL == "" { + return true // Local-only repo → private + } + + urlLower := strings.ToLower(remoteURL) + + // Route to appropriate API handler based on platform + switch { + case strings.Contains(urlLower, "github.com"): + return isPrivateGitHub(remoteURL, httpClient) + case strings.Contains(urlLower, "gitlab"): + return isPrivateGitLab(remoteURL, httpClient) + case strings.Contains(urlLower, "bitbucket"): + return isPrivateBitbucket(remoteURL, httpClient) + case strings.Contains(urlLower, "dev.azure.com") || strings.Contains(urlLower, "visualstudio.com"): + return isPrivateAzureDevOps(remoteURL, httpClient) + default: + return true // Unknown platform → conservative: default to PRIVATE + } +} + +// isPrivateGitHub checks GitHub repo privacy via direct HTTP URL (HTTP 200 = Public, else = Private). +func isPrivateGitHub(repoURL string, httpClient *http.Client) bool { + owner, repo := extractGitHubOwnerRepo(repoURL) + if owner == "" || repo == "" { + return true + } + + directURL := fmt.Sprintf("https://github.com/%s/%s", owner, repo) + isPublic := isRepoPublic(directURL, httpClient) + return !isPublic +} + +// isPrivateGitLab checks gitlab.com repo privacy via HTTP (200 = Public, else = Private); self-hosted URLs default to private for security. +func isPrivateGitLab(repoURL string, httpClient *http.Client) bool { + groupPath, projectName, _ := extractGitLabGroupProject(repoURL) + if groupPath == "" || projectName == "" { + return true + } + + // Hardcode gitlab.com to prevent SSRF attacks via untrusted remote.origin.url + directURL := fmt.Sprintf("https://gitlab.com/%s/%s", groupPath, projectName) + isPublic := isRepoPublic(directURL, httpClient) + return !isPublic +} + +// isPrivateBitbucket checks Bitbucket repo privacy via direct HTTP URL (HTTP 200 = Public, else = Private). +func isPrivateBitbucket(repoURL string, httpClient *http.Client) bool { + workspace, repo := extractBitbucketWorkspaceRepo(repoURL) + if workspace == "" || repo == "" { + return true + } + + directURL := fmt.Sprintf("https://bitbucket.org/%s/%s", workspace, repo) + isPublic := isRepoPublic(directURL, httpClient) + return !isPublic +} + +// isPrivateAzureDevOps checks Azure DevOps repo privacy via public API (no auth required). +func isPrivateAzureDevOps(repoURL string, httpClient *http.Client) bool { + org, repo := extractAzureDevOpsOrgRepo(repoURL) + if org == "" || repo == "" { + return true + } + + directURL := fmt.Sprintf("https://dev.azure.com/%s/_git/%s", org, repo) + isPublic := isRepoPublic(directURL, httpClient) + return !isPublic +} + +// isRepoPublic checks if repository is publicly accessible via injected HTTP client (HTTP 200 = public, else = private). +func isRepoPublic(repoURL string, httpClient *http.Client) bool { + resp, err := httpClient.Get(repoURL) + if err != nil { + logger.PrintIfVerbose(fmt.Sprintf("Repository accessibility check failed for %s: %v, treating as PRIVATE", repoURL, err)) + return false + } + defer func() { + _ = resp.Body.Close() + }() + + isPublic := resp.StatusCode == http.StatusOK + if !isPublic { + logger.PrintIfVerbose(fmt.Sprintf("Repository URL %s returned status %d, treating as PRIVATE", repoURL, resp.StatusCode)) + } + return isPublic +} + +// normalizeSSHURL converts SSH URLs to HTTPS (git@github.com:owner/repo.git → https://github.com/owner/repo.git) +func normalizeSSHURL(repoURL string) string { + if !strings.Contains(repoURL, "://") && strings.Contains(repoURL, "@") && strings.Contains(repoURL, ":") { + // Handle git@host:path format + // git@github.com:owner/repo.git → https://github.com/owner/repo.git + parts := strings.SplitN(repoURL, "@", sshSplitParts) + if len(parts) == sshSplitParts { + hostAndPath := strings.SplitN(parts[1], ":", sshSplitParts) + if len(hostAndPath) == sshSplitParts { + return "https://" + hostAndPath[0] + "/" + hostAndPath[1] + } + } + } else if strings.HasPrefix(repoURL, "ssh://") { + // Handle ssh://git@host/path format → https://host/path + sshURL := strings.TrimPrefix(repoURL, "ssh://") + if strings.Contains(sshURL, "@") { + parts := strings.SplitN(sshURL, "@", sshSplitParts) + if len(parts) == sshSplitParts { + return "https://" + parts[1] + } + } + } + return repoURL +} + +// Extract owner/repo from GitHub URLs: https://github.com/owner/repo or owner/repo +func extractGitHubOwnerRepo(repoURL string) (owner, repo string) { + repoURL = normalizeSSHURL(repoURL) + url := strings.TrimSuffix(repoURL, ".git") + parts := strings.FieldsFunc(url, func(r rune) bool { return r == '/' }) + if len(parts) >= pathParts { + return parts[len(parts)-pathParts], parts[len(parts)-1] + } + return "", "" +} + +// extractGitLabGroupProject extracts group/project from GitLab URLs, supporting nested subgroups (last segment is project, rest is group). +func extractGitLabGroupProject(repoURL string) (group, project, host string) { + repoURL = normalizeSSHURL(repoURL) + url := strings.TrimSuffix(repoURL, ".git") + + // Extract host if present + if strings.Contains(url, "://") { + parts := strings.SplitN(url, "://", urlSchemeParts) + hostAndPath := strings.SplitN(parts[1], "/", pathParts) + if len(hostAndPath) == pathParts { + host = hostAndPath[0] + url = hostAndPath[1] + } + } + + parts := strings.FieldsFunc(url, func(r rune) bool { return r == '/' }) + if len(parts) >= pathParts { + // Take last two segments (project is always the last one; group/subgroup is everything before) + project = parts[len(parts)-1] + group = strings.Join(parts[:len(parts)-1], "/") + return + } + return "", "", host +} + +// Extract workspace/repo from Bitbucket URLs: https://bitbucket.org/workspace/repo +func extractBitbucketWorkspaceRepo(repoURL string) (workspace, repo string) { + repoURL = normalizeSSHURL(repoURL) + url := strings.TrimSuffix(repoURL, ".git") + parts := strings.FieldsFunc(url, func(r rune) bool { return r == '/' }) + if len(parts) >= pathParts { + return parts[len(parts)-pathParts], parts[len(parts)-1] + } + return "", "" +} + +// Extract org/repo from Azure DevOps URLs: https://dev.azure.com/org/_git/repo or https://ssh.dev.azure.com/v3/org/project/repo +func extractAzureDevOpsOrgRepo(repoURL string) (org, repo string) { + repoURL = normalizeSSHURL(repoURL) + url := strings.TrimSuffix(repoURL, ".git") + if strings.Contains(url, "dev.azure.com") { + parts := strings.Split(url, "/") + for i, part := range parts { + if part == "dev.azure.com" && i+1 < len(parts) { + org := parts[i+1] + // Find _git segment (HTTPS format) + for j := i + 2; j < len(parts); j++ { + if parts[j] == "_git" && j+1 < len(parts) { + repo := parts[j+1] + return org, repo + } + } + } else if part == "ssh.dev.azure.com" && i+1 < len(parts) && parts[i+1] == "v3" { + // SSH format: ssh.dev.azure.com/v3/org/project/repo + if i+3 < len(parts) { + org := parts[i+2] + if i+4 < len(parts) { + repo := parts[i+4] + return org, repo + } + } + } + } + } + return "", "" +} + +// fileExists checks if a file exists at the given path. +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} diff --git a/internal/commands/gitmetadata_test.go b/internal/commands/gitmetadata_test.go new file mode 100644 index 000000000..b2907fc41 --- /dev/null +++ b/internal/commands/gitmetadata_test.go @@ -0,0 +1,1503 @@ +package commands + +import ( + "encoding/csv" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// testCommit describes one fixture commit. AllowEmptyCommits means fixtures +// don't need real file content. +type testCommit struct { + email string + name string + when time.Time +} + +func newGitMetadataTestRepo(t *testing.T, remoteURL string, commits []testCommit) string { + t.Helper() + repoPath := t.TempDir() + + repo, err := gogit.PlainInit(repoPath, false) + require.NoError(t, err) + + if remoteURL != "" { + _, err = repo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{remoteURL}, + }) + require.NoError(t, err) + } + + worktree, err := repo.Worktree() + require.NoError(t, err) + + // Commits must be created oldest-first so the last one ends up as HEAD. + for _, c := range commits { + sig := &object.Signature{Name: c.name, Email: c.email, When: c.when} + _, err := worktree.Commit("test commit", &gogit.CommitOptions{ + Author: sig, + AllowEmptyCommits: true, + }) + require.NoError(t, err) + } + + return repoPath +} + +func readGitMetadataFiles(t *testing.T, repoPath string) (csvContent string, metadata contributorsMetadata) { + t.Helper() + csvBytes, err := os.ReadFile(filepath.Join(repoPath, CheckmarxFolderName, ContributorsFileName)) + require.NoError(t, err) + + metadataBytes, err := os.ReadFile(filepath.Join(repoPath, CheckmarxFolderName, MetadataFileName)) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(metadataBytes, &metadata)) + + return string(csvBytes), metadata +} + +func TestGenerateAndWrite_Success(t *testing.T) { + now := time.Now() + repoPath := newGitMetadataTestRepo(t, "https://example.com/org/repo.git", []testCommit{ + {email: "alice@example.com", name: "Alice", when: now.Add(-10 * 24 * time.Hour)}, + {email: "bob@example.com", name: "Bob", when: now.Add(-5 * 24 * time.Hour)}, + {email: "alice@example.com", name: "Alice", when: now.Add(-1 * time.Hour)}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + + csvContent, metadata := readGitMetadataFiles(t, repoPath) + + lines := strings.Split(strings.TrimRight(csvContent, "\n"), "\n") + assert.Len(t, lines, 2, "expected one row per unique email, most recent commit only") + + assert.Equal(t, "https://example.com/org/repo.git", metadata.RepositoryURL) + assert.Equal(t, 3, metadata.CommitsCount, "commitsCount should count every commit in the window, before dedup") + assert.NotEmpty(t, metadata.LastCommitHash) + assert.NotEmpty(t, metadata.LastCommitDate) +} + +func TestGenerateAndWrite_MetadataJSONFieldNames(t *testing.T) { + repoPath := newGitMetadataTestRepo(t, "https://example.com/org/repo.git", []testCommit{ + {email: "alice@example.com", name: "Alice", when: time.Now()}, + }) + require.NoError(t, GenerateAndWrite(repoPath, true)) + + raw, err := os.ReadFile(filepath.Join(repoPath, CheckmarxFolderName, MetadataFileName)) + require.NoError(t, err) + + var asMap map[string]interface{} + require.NoError(t, json.Unmarshal(raw, &asMap)) + + assert.Contains(t, asMap, "repositoryUrl") + assert.Contains(t, asMap, "lastCommitHash") + assert.Contains(t, asMap, "lastCommitDate") + assert.Contains(t, asMap, "commitsCount") + assert.NotContains(t, asMap, "branchName", "branchName is intentionally omitted, see tech design open questions") + assert.NotContains(t, asMap, "commitHash", "field is named lastCommitHash, not commitHash") +} + +func TestGenerateAndWrite_NoHeaderRow(t *testing.T) { + repoPath := newGitMetadataTestRepo(t, "", []testCommit{ + {email: "alice@example.com", name: "Alice", when: time.Now()}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + + csvContent, _ := readGitMetadataFiles(t, repoPath) + firstLine := strings.SplitN(csvContent, ",", 2)[0] + + // A header would read "date" or similar; a real row starts with an RFC3339 + // timestamp, which always begins with a 4-digit year. + _, err := time.Parse(time.RFC3339, firstLine) + assert.NoError(t, err, "first line should be a data row (RFC3339 date), not a header") +} + +func TestGenerateAndWrite_CSVColumnOrder(t *testing.T) { + now := time.Now() + repoPath := newGitMetadataTestRepo(t, "", []testCommit{ + {email: "alice@example.com", name: "Alice Example", when: now}, + }) + require.NoError(t, GenerateAndWrite(repoPath, true)) + + csvContent, metadata := readGitMetadataFiles(t, repoPath) + fields := strings.Split(strings.TrimRight(csvContent, "\n"), ",") + require.Len(t, fields, 4) + + assert.Equal(t, now.Format(time.RFC3339), fields[0], "field 1 must be the commit date") + assert.Equal(t, metadata.LastCommitHash, fields[1], "field 2 must be the commit hash") + assert.Equal(t, "alice@example.com", fields[2], "field 3 must be the email") + assert.Equal(t, "Alice Example", fields[3], "field 4 must be the username") +} + +func TestGenerateAndWrite_DedupKeepsMostRecentPerEmail(t *testing.T) { + now := time.Now() + olderTime := now.Add(-20 * 24 * time.Hour) + newerTime := now.Add(-1 * 24 * time.Hour) + + repoPath := newGitMetadataTestRepo(t, "", []testCommit{ + {email: "alice@example.com", name: "Alice Old", when: olderTime}, + {email: "alice@example.com", name: "Alice New", when: newerTime}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + + csvContent, _ := readGitMetadataFiles(t, repoPath) + lines := strings.Split(strings.TrimRight(csvContent, "\n"), "\n") + require.Len(t, lines, 1) + assert.Contains(t, lines[0], newerTime.Format(time.RFC3339)) + assert.Contains(t, lines[0], "Alice New") + assert.NotContains(t, lines[0], "Alice Old") +} + +func TestGenerateAndWrite_DedupIsCaseInsensitiveOnEmail(t *testing.T) { + now := time.Now() + repoPath := newGitMetadataTestRepo(t, "", []testCommit{ + {email: "Alice@Example.com", name: "Alice Mixed Case", when: now.Add(-2 * 24 * time.Hour)}, + {email: "alice@example.com", name: "Alice Lower Case", when: now.Add(-1 * time.Hour)}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + + csvContent, metadata := readGitMetadataFiles(t, repoPath) + lines := strings.Split(strings.TrimRight(csvContent, "\n"), "\n") + assert.Len(t, lines, 1, "differently-cased emails for the same person should dedup to one row") + assert.Equal(t, 2, metadata.CommitsCount, "commitsCount still counts both raw commits") +} + +func TestGenerateAndWrite_ManyUniqueEmailsAllKept(t *testing.T) { + now := time.Now() + var commits []testCommit + for i := 0; i < 5; i++ { + commits = append(commits, testCommit{ + email: strings.Repeat("u", 1) + string(rune('a'+i)) + "@example.com", + name: "User", + when: now.Add(-time.Duration(i) * time.Hour), + }) + } + repoPath := newGitMetadataTestRepo(t, "", commits) + require.NoError(t, GenerateAndWrite(repoPath, true)) + + csvContent, metadata := readGitMetadataFiles(t, repoPath) + lines := strings.Split(strings.TrimRight(csvContent, "\n"), "\n") + assert.Len(t, lines, 5, "each unique email should get its own row") + assert.Equal(t, 5, metadata.CommitsCount) +} + +func TestGenerateAndWrite_90DayBoundary(t *testing.T) { + now := time.Now() + repoPath := newGitMetadataTestRepo(t, "", []testCommit{ + {email: "old@example.com", name: "TooOld", when: now.Add(-91 * 24 * time.Hour)}, + {email: "recent@example.com", name: "Recent", when: now.Add(-89 * 24 * time.Hour)}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + + csvContent, metadata := readGitMetadataFiles(t, repoPath) + assert.Contains(t, csvContent, "recent@example.com") + assert.NotContains(t, csvContent, "old@example.com") + assert.Equal(t, 1, metadata.CommitsCount) +} + +func TestGenerateAndWrite_AllCommitsOutsideWindow(t *testing.T) { + now := time.Now() + repoPath := newGitMetadataTestRepo(t, "", []testCommit{ + {email: "old@example.com", name: "TooOld", when: now.Add(-200 * 24 * time.Hour)}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + + csvContent, metadata := readGitMetadataFiles(t, repoPath) + assert.Empty(t, csvContent, "no commits in the window means an empty (but present) CSV") + assert.Equal(t, 0, metadata.CommitsCount) + assert.NotEmpty(t, metadata.LastCommitHash, "HEAD info is unconditional, independent of the 90-day window") +} + +func TestGenerateAndWrite_NoRemote(t *testing.T) { + repoPath := newGitMetadataTestRepo(t, "", []testCommit{ + {email: "alice@example.com", name: "Alice", when: time.Now()}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + + _, metadata := readGitMetadataFiles(t, repoPath) + assert.Empty(t, metadata.RepositoryURL) +} + +func TestGenerateAndWrite_SSHRemoteURL(t *testing.T) { + repoPath := newGitMetadataTestRepo(t, "git@github.com:org/repo.git", []testCommit{ + {email: "alice@example.com", name: "Alice", when: time.Now()}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + + _, metadata := readGitMetadataFiles(t, repoPath) + assert.Equal(t, "git@github.com:org/repo.git", metadata.RepositoryURL) +} + +func TestGenerateAndWrite_PublicRepoNoCSV(t *testing.T) { + repoPath := newGitMetadataTestRepo(t, "https://example.com/org/repo.git", []testCommit{ + {email: "alice@example.com", name: "Alice", when: time.Now()}, + }) + + // Call with isPrivateRepo = false (public repo) + require.NoError(t, GenerateAndWrite(repoPath, false)) + + // Check that CSV does NOT exist (public repo should not have contributors.csv) + csvPath := filepath.Join(repoPath, CheckmarxFolderName, ContributorsFileName) + _, err := os.ReadFile(csvPath) + assert.Error(t, err, "public repo should NOT generate contributors.csv") + assert.True(t, os.IsNotExist(err), "CSV file should not exist for public repo") + + // But metadata.json should still exist + metadataPath := filepath.Join(repoPath, CheckmarxFolderName, MetadataFileName) + metadataBytes, err := os.ReadFile(metadataPath) + require.NoError(t, err, "public repo should still generate metadata.json") + + var metadata contributorsMetadata + err = json.Unmarshal(metadataBytes, &metadata) + require.NoError(t, err) + + assert.NotEmpty(t, metadata.RepositoryURL, "metadata should contain repository URL") + assert.NotEmpty(t, metadata.LastCommitHash, "metadata should contain commit hash") + assert.NotEmpty(t, metadata.LastCommitDate, "metadata should contain commit date") +} + +func TestGenerateAndWrite_ReplacesStaleFiles(t *testing.T) { + repoPath := newGitMetadataTestRepo(t, "", []testCommit{ + {email: "alice@example.com", name: "Alice", when: time.Now()}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + _, firstMetadata := readGitMetadataFiles(t, repoPath) + + repo, err := gogit.PlainOpen(repoPath) + require.NoError(t, err) + worktree, err := repo.Worktree() + require.NoError(t, err) + _, err = worktree.Commit("second commit", &gogit.CommitOptions{ + Author: &object.Signature{Name: "Bob", Email: "bob@example.com", When: time.Now()}, + AllowEmptyCommits: true, + }) + require.NoError(t, err) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + csvContent, secondMetadata := readGitMetadataFiles(t, repoPath) + + assert.NotEqual(t, firstMetadata.LastCommitHash, secondMetadata.LastCommitHash, "second run should overwrite metadata.json with fresh data") + assert.Contains(t, csvContent, "bob@example.com") + assert.Contains(t, csvContent, "alice@example.com") +} + +func TestGenerateAndWrite_RunTwiceIdenticalStateProducesIdenticalOutput(t *testing.T) { + repoPath := newGitMetadataTestRepo(t, "https://example.com/repo.git", []testCommit{ + {email: "alice@example.com", name: "Alice", when: time.Now()}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + firstCSV, firstMetadata := readGitMetadataFiles(t, repoPath) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + secondCSV, secondMetadata := readGitMetadataFiles(t, repoPath) + + assert.Equal(t, firstCSV, secondCSV) + assert.Equal(t, firstMetadata, secondMetadata) +} + +func TestGenerateAndWrite_NotAGitRepository(t *testing.T) { + dir := t.TempDir() + err := GenerateAndWrite(dir, true) + assert.Error(t, err) +} + +func TestGenerateAndWrite_NoCommits(t *testing.T) { + dir := t.TempDir() + _, err := gogit.PlainInit(dir, false) + require.NoError(t, err) + + err = GenerateAndWrite(dir, true) + assert.Error(t, err, "an empty repo has no HEAD to resolve") +} + +func TestGenerateAndWrite_NonExistentPath(t *testing.T) { + err := GenerateAndWrite(filepath.Join(t.TempDir(), "does-not-exist"), true) + assert.Error(t, err) +} + +// TestBuildContributorsCSV_ExceedsSizeLimitStillSucceeds exercises buildContributorsCSV +// directly with synthetic in-memory commits (no real git repo), since creating enough +// real commits to exceed the 1MB warning threshold would make the test very slow. +func TestBuildContributorsCSV_ExceedsSizeLimitStillSucceeds(t *testing.T) { + now := time.Now() + commits := make([]*object.Commit, 0, 20000) + for i := 0; i < 20000; i++ { + commits = append(commits, &object.Commit{ + Author: object.Signature{ + Name: "User", + Email: "user" + strconv.Itoa(i) + "@example.com", + When: now.Add(-time.Duration(i) * time.Second), + }, + }) + } + + csvData, err := buildContributorsCSV(commits) + require.NoError(t, err, "exceeding the size warning threshold must not fail generation") + assert.Greater(t, len(csvData), contributorsSizeLimit) + + lines := strings.Split(strings.TrimRight(string(csvData), "\n"), "\n") + assert.Len(t, lines, len(commits), "all unique emails should be kept as rows") +} + +func TestBuildContributorsCSV_EmptyInput(t *testing.T) { + csvData, err := buildContributorsCSV(nil) + require.NoError(t, err) + assert.Empty(t, csvData) +} + +func TestDedupByEmail_EmptyInput(t *testing.T) { + assert.Empty(t, dedupByEmail(nil)) +} + +func TestDedupByEmail_PreservesFirstOccurrenceOrder(t *testing.T) { + now := time.Now() + commits := []*object.Commit{ + {Author: object.Signature{Email: "a@example.com", When: now}}, + {Author: object.Signature{Email: "b@example.com", When: now}}, + {Author: object.Signature{Email: "a@example.com", When: now}}, // duplicate, later in slice + {Author: object.Signature{Email: "c@example.com", When: now}}, + } + + deduped := dedupByEmail(commits) + require.Len(t, deduped, 3) + assert.Equal(t, "a@example.com", deduped[0].Author.Email) + assert.Equal(t, "b@example.com", deduped[1].Author.Email) + assert.Equal(t, "c@example.com", deduped[2].Author.Email) +} + +func TestBuildMetadataJSON_Structure(t *testing.T) { + headCommit := &object.Commit{ + Author: object.Signature{When: time.Date(2025, 9, 30, 10, 35, 5, 0, time.UTC)}, + } + + data, err := buildMetadataJSON("https://example.com/repo.git", headCommit, 42) + require.NoError(t, err) + + var asMap map[string]interface{} + require.NoError(t, json.Unmarshal(data, &asMap)) + assert.Equal(t, "https://example.com/repo.git", asMap["repositoryUrl"]) + assert.Equal(t, headCommit.Hash.String(), asMap["lastCommitHash"]) + assert.Equal(t, "2025-09-30T10:35:05Z", asMap["lastCommitDate"]) + assert.InDelta(t, 42, asMap["commitsCount"], 0) +} + +func TestCommitsSince_SortsNewestFirst(t *testing.T) { + now := time.Now() + // Committed out of chronological order to verify commitsSince re-sorts them. + repoPath := newGitMetadataTestRepo(t, "", []testCommit{ + {email: "a@example.com", name: "A", when: now.Add(-5 * 24 * time.Hour)}, + {email: "b@example.com", name: "B", when: now.Add(-1 * 24 * time.Hour)}, + {email: "c@example.com", name: "C", when: now.Add(-10 * 24 * time.Hour)}, + }) + + repo, err := gogit.PlainOpen(repoPath) + require.NoError(t, err) + + commits, err := commitsSince(repo, now.Add(-30*24*time.Hour)) + require.NoError(t, err) + require.Len(t, commits, 3) + + assert.True(t, commits[0].Author.When.After(commits[1].Author.When)) + assert.True(t, commits[1].Author.When.After(commits[2].Author.When)) +} + +func TestRemoteURL_ReturnsFirstURL(t *testing.T) { + repoPath := newGitMetadataTestRepo(t, "https://example.com/first.git", nil) + repo, err := gogit.PlainOpen(repoPath) + require.NoError(t, err) + + assert.Equal(t, "https://example.com/first.git", remoteURL(repo)) +} + +func TestRemoteURL_NoRemoteConfigured(t *testing.T) { + repoPath := newGitMetadataTestRepo(t, "", nil) + repo, err := gogit.PlainOpen(repoPath) + require.NoError(t, err) + + assert.Empty(t, remoteURL(repo)) +} + +func TestParseGitLogOutput_BasicParsing(t *testing.T) { + logOutput := "2026-08-19T18:24:26+05:30\x1fabc123def456789abc123def456789abc12345\x1fuser@example.com\x1fJohn Doe\n" + + "2026-08-13T12:58:25+03:00\x1fdef456789abc123def456789abc123def45678\x1fjane@example.com\x1fJane Smith" + + commits := parseGitLogOutput(logOutput, time.Time{}) + require.Len(t, commits, 2) + + // Git log returns newest-first; commits[0] should be the newer commit + assert.Equal(t, "2026-08-19T18:24:26+05:30", commits[0]["date"]) + assert.Equal(t, "abc123def456789abc123def456789abc12345", commits[0]["hash"]) + assert.Equal(t, "user@example.com", commits[0]["email"]) + assert.Equal(t, "John Doe", commits[0]["name"]) + + assert.Equal(t, "2026-08-13T12:58:25+03:00", commits[1]["date"]) + assert.Equal(t, "def456789abc123def456789abc123def45678", commits[1]["hash"]) + assert.Equal(t, "jane@example.com", commits[1]["email"]) + assert.Equal(t, "Jane Smith", commits[1]["name"]) +} + +func TestParseGitLogOutput_ReturnsNewestFirst(t *testing.T) { + // Git log returns commits newest-first by default; test input reflects actual git output + logOutput := "2026-08-19T20:00:00+00:00\x1f3333333333333333333333333333333333333333\x1fnew@example.com\x1fNew User\n" + + "2026-07-15T15:00:00+00:00\x1f2222222222222222222222222222222222222222\x1fmid@example.com\x1fMid User\n" + + "2026-07-01T10:00:00+00:00\x1f1111111111111111111111111111111111111111\x1fold@example.com\x1fOld User" + + commits := parseGitLogOutput(logOutput, time.Time{}) + require.Len(t, commits, 3) + + // Should preserve git log's newest-first order + assert.Equal(t, "2026-08-19T20:00:00+00:00", commits[0]["date"]) + assert.Equal(t, "2026-07-15T15:00:00+00:00", commits[1]["date"]) + assert.Equal(t, "2026-07-01T10:00:00+00:00", commits[2]["date"]) +} + +func TestParseGitLogOutput_EmptyInput(t *testing.T) { + commits := parseGitLogOutput("", time.Time{}) + assert.Nil(t, commits) +} + +func TestParseGitLogOutput_SkipsMalformedLines(t *testing.T) { + logOutput := "2026-08-19T18:24:26+05:30\x1fabc123def456789abc123def456789abc12345\x1fuser@example.com\x1fJohn Doe\n" + + "malformed line\n" + + "2026-08-13T12:58:25+03:00\x1fdef456789abc123def456789abc123def45678\x1fjane@example.com\x1fJane Smith\n" + + "\n" + + "another bad line" + + commits := parseGitLogOutput(logOutput, time.Time{}) + require.Len(t, commits, 2) + // Git log returns newest-first; John Doe (2026-08-19) before Jane Smith (2026-08-13) + assert.Equal(t, "John Doe", commits[0]["name"]) + assert.Equal(t, "Jane Smith", commits[1]["name"]) +} + +func TestConvertToCoreCommits_BasicConversion(t *testing.T) { + now := time.Now() + commitMaps := []map[string]string{ + { + "date": now.Format(time.RFC3339), + "hash": "abc123def456789abcdef456789abcdef1234567", + "email": "user@example.com", + "name": "Test User", + }, + } + + commits := convertToCoreCommits(commitMaps) + require.Len(t, commits, 1) + + c := commits[0] + assert.Equal(t, "abc123def456789abcdef456789abcdef1234567", c.Hash.String()) + assert.Equal(t, "user@example.com", c.Author.Email) + assert.Equal(t, "Test User", c.Author.Name) + assert.WithinDuration(t, now, c.Author.When, 1*time.Second) +} + +func TestConvertToCoreCommits_PreservesOrder(t *testing.T) { + commitMaps := []map[string]string{ + { + "date": "2026-08-19T18:24:26+05:30", + "hash": "1111111111111111111111111111111111111111", + "email": "first@example.com", + "name": "First", + }, + { + "date": "2026-08-13T12:58:25+03:00", + "hash": "2222222222222222222222222222222222222222", + "email": "second@example.com", + "name": "Second", + }, + } + + commits := convertToCoreCommits(commitMaps) + require.Len(t, commits, 2) + + assert.Equal(t, "1111111111111111111111111111111111111111", commits[0].Hash.String()) + assert.Equal(t, "2222222222222222222222222222222222222222", commits[1].Hash.String()) +} + +func TestConvertToCoreCommits_RFC3339DateParsing(t *testing.T) { + commitMaps := []map[string]string{ + { + "date": "2026-08-19T18:24:26+05:30", + "hash": "abc123", + "email": "user@example.com", + "name": "User", + }, + } + + commits := convertToCoreCommits(commitMaps) + require.Len(t, commits, 1) + + // Verify the date was parsed correctly (RFC3339 with timezone) + assert.Equal(t, 2026, commits[0].Author.When.Year()) + assert.Equal(t, time.August, commits[0].Author.When.Month()) + assert.Equal(t, 19, commits[0].Author.When.Day()) +} + +func TestBuildMetadataJSONFromSystem_Structure(t *testing.T) { + data, err := buildMetadataJSONFromSystem("https://github.com/example/repo.git", "abc123def456", 42, "2026-08-19T18:24:26Z") + require.NoError(t, err) + + var asMap map[string]interface{} + require.NoError(t, json.Unmarshal(data, &asMap)) + + assert.Equal(t, "https://github.com/example/repo.git", asMap["repositoryUrl"]) + assert.Equal(t, "abc123def456", asMap["lastCommitHash"]) + assert.InDelta(t, 42, asMap["commitsCount"], 0) + assert.NotEmpty(t, asMap["lastCommitDate"]) + + // Verify lastCommitDate is valid RFC3339 + _, err = time.Parse(time.RFC3339, asMap["lastCommitDate"].(string)) + require.NoError(t, err) +} + +func TestBuildMetadataJSONFromSystem_EmptyRepository(t *testing.T) { + data, err := buildMetadataJSONFromSystem("", "", 0, "") + require.NoError(t, err) + + var asMap map[string]interface{} + require.NoError(t, json.Unmarshal(data, &asMap)) + + assert.Equal(t, "", asMap["repositoryUrl"]) + assert.Equal(t, "", asMap["lastCommitHash"]) + assert.InDelta(t, 0, asMap["commitsCount"], 0) +} + +func TestGitCommand_Success(t *testing.T) { + // Create a real git repo for this test + repoPath := t.TempDir() + + // Initialize a git repository using system git + cmd := exec.Command("git", "init") + cmd.Dir = repoPath + require.NoError(t, cmd.Run()) + + // Configure git user for this test repo (required in CI) + cmd = exec.Command("git", "config", "user.name", "Test User") + cmd.Dir = repoPath + require.NoError(t, cmd.Run()) + + // Test git config command + output, err := gitCommand(repoPath, "config", "--get", "user.name") + assert.NoError(t, err) + assert.Equal(t, "Test User", output) +} + +func TestGitCommand_InvalidRepo(t *testing.T) { + invalidPath := t.TempDir() + // This directory is not a git repo + + _, err := gitCommand(invalidPath, "log", "--oneline") + assert.Error(t, err) + assert.Contains(t, err.Error(), "git command failed") +} + +// Tests for URL extraction functions + +func TestExtractGitHubOwnerRepo(t *testing.T) { + tests := []struct { + name string + url string + wantOwner string + wantRepo string + }{ + { + name: "HTTPS GitHub URL", + url: "https://github.com/checkmarx/ast-cli", + wantOwner: "checkmarx", + wantRepo: "ast-cli", + }, + { + name: "HTTPS GitHub URL with .git suffix", + url: "https://github.com/checkmarx/ast-cli.git", + wantOwner: "checkmarx", + wantRepo: "ast-cli", + }, + { + name: "Short format GitHub URL", + url: "checkmarx/ast-cli", + wantOwner: "checkmarx", + wantRepo: "ast-cli", + }, + { + name: "Invalid format - too few parts", + url: "invalid", + wantOwner: "", + wantRepo: "", + }, + { + name: "Empty URL", + url: "", + wantOwner: "", + wantRepo: "", + }, + { + name: "SSH GitHub URL", + url: "git@github.com:checkmarx/ast-cli.git", + wantOwner: "checkmarx", + wantRepo: "ast-cli", + }, + { + name: "SSH GitHub URL without .git", + url: "git@github.com:checkmarx/ast-cli", + wantOwner: "checkmarx", + wantRepo: "ast-cli", + }, + { + name: "SSH protocol GitHub URL", + url: "ssh://git@github.com/checkmarx/ast-cli.git", + wantOwner: "checkmarx", + wantRepo: "ast-cli", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + owner, repo := extractGitHubOwnerRepo(tt.url) + assert.Equal(t, tt.wantOwner, owner, "owner mismatch") + assert.Equal(t, tt.wantRepo, repo, "repo mismatch") + }) + } +} + +func TestExtractGitLabGroupProject(t *testing.T) { + tests := []struct { + name string + url string + wantGroup string + wantRepo string + wantHost string + }{ + { + name: "HTTPS GitLab URL", + url: "https://gitlab.com/checkmarx/ast-cli", + wantGroup: "checkmarx", + wantRepo: "ast-cli", + wantHost: "gitlab.com", + }, + { + name: "HTTPS GitLab URL with .git suffix", + url: "https://gitlab.com/checkmarx/ast-cli.git", + wantGroup: "checkmarx", + wantRepo: "ast-cli", + wantHost: "gitlab.com", + }, + { + name: "Self-hosted GitLab", + url: "https://gitlab.internal.com/checkmarx/ast-cli", + wantGroup: "checkmarx", + wantRepo: "ast-cli", + wantHost: "gitlab.internal.com", + }, + { + name: "SSH GitLab URL", + url: "git@gitlab.com:checkmarx/ast-cli.git", + wantGroup: "checkmarx", + wantRepo: "ast-cli", + wantHost: "gitlab.com", + }, + { + name: "SSH self-hosted GitLab URL", + url: "git@gitlab.internal.com:checkmarx/ast-cli.git", + wantGroup: "checkmarx", + wantRepo: "ast-cli", + wantHost: "gitlab.internal.com", + }, + { + name: "nested subgroup (two levels)", + url: "https://gitlab.com/myorg/myteam/myproject", + wantGroup: "myorg/myteam", + wantRepo: "myproject", + wantHost: "gitlab.com", + }, + { + name: "nested subgroup (three levels)", + url: "https://gitlab.com/org/team/platform/repo", + wantGroup: "org/team/platform", + wantRepo: "repo", + wantHost: "gitlab.com", + }, + { + name: "nested subgroup with .git suffix", + url: "https://gitlab.com/org/sub1/sub2/project.git", + wantGroup: "org/sub1/sub2", + wantRepo: "project", + wantHost: "gitlab.com", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + group, project, host := extractGitLabGroupProject(tt.url) + assert.Equal(t, tt.wantGroup, group, "group mismatch") + assert.Equal(t, tt.wantRepo, project, "project mismatch") + assert.Equal(t, tt.wantHost, host, "host mismatch") + }) + } +} + +func TestExtractBitbucketWorkspaceRepo(t *testing.T) { + tests := []struct { + name string + url string + wantWorkspace string + wantRepo string + }{ + { + name: "HTTPS Bitbucket URL", + url: "https://bitbucket.org/checkmarx/ast-cli", + wantWorkspace: "checkmarx", + wantRepo: "ast-cli", + }, + { + name: "HTTPS Bitbucket URL with .git suffix", + url: "https://bitbucket.org/checkmarx/ast-cli.git", + wantWorkspace: "checkmarx", + wantRepo: "ast-cli", + }, + { + name: "SSH Bitbucket URL", + url: "git@bitbucket.org:checkmarx/ast-cli.git", + wantWorkspace: "checkmarx", + wantRepo: "ast-cli", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + workspace, repo := extractBitbucketWorkspaceRepo(tt.url) + assert.Equal(t, tt.wantWorkspace, workspace, "workspace mismatch") + assert.Equal(t, tt.wantRepo, repo, "repo mismatch") + }) + } +} + +func TestExtractAzureDevOpsOrgRepo(t *testing.T) { + tests := []struct { + name string + url string + wantOrg string + wantRepo string + }{ + { + name: "HTTPS Azure DevOps URL with _git", + url: "https://dev.azure.com/checkmarx/project/_git/ast-cli", + wantOrg: "checkmarx", + wantRepo: "ast-cli", + }, + { + name: "HTTPS Azure DevOps URL with .git suffix", + url: "https://dev.azure.com/checkmarx/project/_git/ast-cli.git", + wantOrg: "checkmarx", + wantRepo: "ast-cli", + }, + { + name: "SSH Azure DevOps URL", + url: "git@ssh.dev.azure.com:v3/checkmarx/project/ast-cli.git", + wantOrg: "checkmarx", + wantRepo: "ast-cli", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + org, repo := extractAzureDevOpsOrgRepo(tt.url) + assert.Equal(t, tt.wantOrg, org, "org mismatch") + assert.Equal(t, tt.wantRepo, repo, "repo mismatch") + }) + } +} + +func TestNormalizeSSHURL(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "git@github.com format", + input: "git@github.com:owner/repo.git", + expected: "https://github.com/owner/repo.git", + }, + { + name: "git@gitlab.com with nested path", + input: "git@gitlab.com:org/team/project.git", + expected: "https://gitlab.com/org/team/project.git", + }, + { + name: "ssh:// URL with git@", + input: "ssh://git@github.com/owner/repo.git", + expected: "https://github.com/owner/repo.git", + }, + { + name: "ssh:// URL with gitlab", + input: "ssh://git@gitlab.com/org/project.git", + expected: "https://gitlab.com/org/project.git", + }, + { + name: "already HTTPS URL", + input: "https://github.com/owner/repo.git", + expected: "https://github.com/owner/repo.git", + }, + { + name: "HTTP URL", + input: "http://github.com/owner/repo.git", + expected: "http://github.com/owner/repo.git", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := normalizeSSHURL(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} + +// Tests for privacy detection with mocked HTTP responses + +func TestFileExists(t *testing.T) { + tests := []struct { + name string + setup func(t *testing.T) string + wantTrue bool + }{ + { + name: "file exists", + setup: func(t *testing.T) string { + f, err := os.CreateTemp(t.TempDir(), "test") + require.NoError(t, err) + path := f.Name() + require.NoError(t, f.Close()) + return path + }, + wantTrue: true, + }, + { + name: "file does not exist", + setup: func(t *testing.T) string { + return "/nonexistent/path/that/does/not/exist" + }, + wantTrue: false, + }, + { + name: "directory exists", + setup: func(t *testing.T) string { + return t.TempDir() + }, + wantTrue: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := tt.setup(t) + exists := fileExists(path) + assert.Equal(t, tt.wantTrue, exists) + }) + } +} + +func TestPrivacyDetectionWithMockedHTTP(t *testing.T) { + mockClient := &http.Client{Timeout: 5 * time.Second} + + t.Run("detectRepositoryPrivacy returns private for empty path", func(t *testing.T) { + tmpDir := t.TempDir() + // Empty directory with no .git or .checkmarx files + isPrivate := detectRepositoryPrivacy(tmpDir, mockClient) + assert.True(t, isPrivate, "empty directory should default to private") + }) + + t.Run("detectRepositoryPrivacy returns private when checking nonexistent path", func(t *testing.T) { + isPrivate := detectRepositoryPrivacy("/nonexistent/path/that/does/not/exist", mockClient) + assert.True(t, isPrivate, "nonexistent path should default to private") + }) + + t.Run("isPrivateByURL returns private for empty URL", func(t *testing.T) { + isPrivate := isPrivateByURL("", mockClient) + assert.True(t, isPrivate, "empty URL should be private") + }) + + t.Run("isPrivateByURL routes unknown platforms to private", func(t *testing.T) { + isPrivate := isPrivateByURL("https://unknown-git-hosting.com/team/repo", mockClient) + assert.True(t, isPrivate, "unknown platform should default to private") + }) +} + +func TestIsRepoPublicWithMockedServer(t *testing.T) { + mockClient := &http.Client{Timeout: 5 * time.Second} + + t.Run("public repository returns true when HTTP 200", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + // Test with mocked server URL + isPublic := isRepoPublic(server.URL, mockClient) + assert.True(t, isPublic, "HTTP 200 should indicate public") + }) + + t.Run("private repository returns false when HTTP 404", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + isPublic := isRepoPublic(server.URL, mockClient) + assert.False(t, isPublic, "HTTP 404 should indicate not public (private)") + }) + + t.Run("empty URL returns false (not public/private)", func(t *testing.T) { + isPublic := isRepoPublic("", mockClient) + assert.False(t, isPublic, "empty URL should not be public") + }) + + t.Run("malformed URL returns false (not public/private)", func(t *testing.T) { + isPublic := isRepoPublic("://invalid", mockClient) + assert.False(t, isPublic, "malformed URL should not be public") + }) +} + +func TestPlatformSpecificPrivacyDetection(t *testing.T) { + mockClient := &http.Client{Timeout: 5 * time.Second} + + t.Run("isPrivateGitHub returns private when extraction fails", func(t *testing.T) { + // Invalid URL that won't extract properly - returns early without HTTP call + isPrivate := isPrivateGitHub("invalid", mockClient) + assert.True(t, isPrivate, "invalid URL should be private") + }) + + t.Run("isPrivateGitHub returns private for empty owner", func(t *testing.T) { + // URL format that extracts to empty owner + isPrivate := isPrivateGitHub("", mockClient) + assert.True(t, isPrivate, "empty URL should be private") + }) + + t.Run("isPrivateGitLab returns private when extraction fails", func(t *testing.T) { + // Invalid URL that won't extract properly + isPrivate := isPrivateGitLab("invalid", mockClient) + assert.True(t, isPrivate, "invalid URL should be private") + }) + + t.Run("isPrivateGitLab returns private for empty group", func(t *testing.T) { + // URL format that extracts to empty group + isPrivate := isPrivateGitLab("", mockClient) + assert.True(t, isPrivate, "empty URL should be private") + }) + + t.Run("isPrivateBitbucket returns private when extraction fails", func(t *testing.T) { + // Invalid URL that won't extract properly + isPrivate := isPrivateBitbucket("invalid", mockClient) + assert.True(t, isPrivate, "invalid URL should be private") + }) + + t.Run("isPrivateAzureDevOps returns private when extraction fails", func(t *testing.T) { + // Invalid URL that won't extract properly + isPrivate := isPrivateAzureDevOps("invalid", mockClient) + assert.True(t, isPrivate, "invalid URL should be private") + }) + + t.Run("isPrivateBitbucket returns private for empty URL", func(t *testing.T) { + isPrivate := isPrivateBitbucket("", mockClient) + assert.True(t, isPrivate, "empty URL should be private") + }) + + t.Run("isPrivateAzureDevOps returns private for empty URL", func(t *testing.T) { + isPrivate := isPrivateAzureDevOps("", mockClient) + assert.True(t, isPrivate, "empty URL should be private") + }) +} + +func TestExtractionsReturnEmptyOnInvalidInput(t *testing.T) { + t.Run("extractGitHubOwnerRepo with empty URL", func(t *testing.T) { + owner, repo := extractGitHubOwnerRepo("") + assert.Equal(t, "", owner) + assert.Equal(t, "", repo) + }) + + t.Run("extractGitLabGroupProject with empty URL", func(t *testing.T) { + group, project, host := extractGitLabGroupProject("") + assert.Equal(t, "", group) + assert.Equal(t, "", project) + assert.Equal(t, "", host) + }) + + t.Run("extractBitbucketWorkspaceRepo with empty URL", func(t *testing.T) { + workspace, repo := extractBitbucketWorkspaceRepo("") + assert.Equal(t, "", workspace) + assert.Equal(t, "", repo) + }) + + t.Run("extractAzureDevOpsOrgRepo with empty URL", func(t *testing.T) { + org, repo := extractAzureDevOpsOrgRepo("") + assert.Equal(t, "", org) + assert.Equal(t, "", repo) + }) + + t.Run("extractGitHubOwnerRepo with single segment", func(t *testing.T) { + owner, repo := extractGitHubOwnerRepo("onlysegment") + assert.Equal(t, "", owner) + assert.Equal(t, "", repo) + }) + + t.Run("extractBitbucketWorkspaceRepo with single segment", func(t *testing.T) { + workspace, repo := extractBitbucketWorkspaceRepo("onlysegment") + assert.Equal(t, "", workspace) + assert.Equal(t, "", repo) + }) +} + +func TestParseGitLogOutput_FiltersByAuthorDate(t *testing.T) { + now := time.Now() + cutoff := now.Add(-10 * 24 * time.Hour) + + // Log output with commits before and after cutoff + logOutput := fmt.Sprintf( + "%s\x1f%s\x1f%s\x1f%s\n%s\x1f%s\x1f%s\x1f%s", + cutoff.Add(-time.Hour).Format(time.RFC3339), "hash1", "old@example.com", "Old User", + cutoff.Add(24*time.Hour).Format(time.RFC3339), "hash2", "new@example.com", "New User", + ) + + commits := parseGitLogOutput(logOutput, cutoff) + // Only the commit after cutoff should be included + assert.Len(t, commits, 1) + assert.Equal(t, "new@example.com", commits[0]["email"]) +} + +func TestParseGitLogOutput_SkipsMalformedDates(t *testing.T) { + logOutput := fmt.Sprintf( + "invalid-date\x1f%s\x1f%s\x1f%s\n%s\x1f%s\x1f%s\x1f%s", + "hash1", "user1@example.com", "User 1", + time.Now().Format(time.RFC3339), "hash2", "user2@example.com", "User 2", + ) + + cutoff := time.Now().Add(-24 * time.Hour) + commits := parseGitLogOutput(logOutput, cutoff) + // Should skip the malformed date line and only include the valid one + assert.Len(t, commits, 1) + assert.Equal(t, "user2@example.com", commits[0]["email"]) +} + +func TestParseGitLogOutput_SkipsIncompleteLines(t *testing.T) { + logOutput := fmt.Sprintf( + "%s\x1f%s\x1f%s\n%s\x1f%s\x1f%s\x1f%s", + time.Now().Format(time.RFC3339), "hash1", "incomplete", + time.Now().Format(time.RFC3339), "hash2", "user2@example.com", "User 2", + ) + + commits := parseGitLogOutput(logOutput, time.Now().Add(-24*time.Hour)) + // Should skip the incomplete line (only 3 fields instead of 4) + assert.Len(t, commits, 1) + assert.Equal(t, "user2@example.com", commits[0]["email"]) +} + +func TestConvertToCoreCommits_HandlesParsingErrors(t *testing.T) { + commits := []map[string]string{ + { + "date": "invalid-date", + "hash": "0000000000000000000000000000000000000000", + "email": "user@example.com", + "name": "User", + }, + { + "date": time.Now().Format(time.RFC3339), + "hash": "1234567890123456789012345678901234567890", + "email": "user2@example.com", + "name": "User 2", + }, + } + + result := convertToCoreCommits(commits) + assert.Len(t, result, 2) + // First commit should have zero time due to parsing error + assert.True(t, result[0].Author.When.IsZero()) + // Second commit should parse correctly + assert.False(t, result[1].Author.When.IsZero()) +} + +func TestWriteGeneratedFilesConditional_CreatesCheckmarxDir(t *testing.T) { + repoPath := t.TempDir() + csvData := []byte("date,hash,email,name\n2026-09-17T00:00:00Z,abc123,user@example.com,Test User") + metadataData := []byte(`{"repositoryUrl":"https://github.com/test/repo","lastCommitHash":"abc123","lastCommitDate":"2026-09-17T00:00:00Z","commitsCount":1}`) + + // Private repo should write both files + err := writeGeneratedFilesConditional(repoPath, csvData, metadataData, true) + require.NoError(t, err) + + // Check .checkmarx folder was created + checkmarxDir := filepath.Join(repoPath, CheckmarxFolderName) + assert.True(t, fileExists(checkmarxDir), "should create .checkmarx directory") + + // Check both files exist + csvPath := filepath.Join(checkmarxDir, ContributorsFileName) + metadataPath := filepath.Join(checkmarxDir, MetadataFileName) + assert.True(t, fileExists(csvPath), "should create contributors.csv for private repo") + assert.True(t, fileExists(metadataPath), "should create metadata.json") + + // Verify file contents + csvBytes, err := os.ReadFile(csvPath) + require.NoError(t, err) + assert.Equal(t, string(csvData), string(csvBytes)) +} + +func TestWriteGeneratedFilesConditional_PublicRepoNoCSV(t *testing.T) { + repoPath := t.TempDir() + csvData := []byte("date,hash,email,name\n2026-09-17T00:00:00Z,abc123,user@example.com,Test User") + metadataData := []byte(`{"repositoryUrl":"https://github.com/test/repo","lastCommitHash":"abc123","lastCommitDate":"2026-09-17T00:00:00Z","commitsCount":1}`) + + // Public repo (isPrivateRepo=false) should NOT write CSV + err := writeGeneratedFilesConditional(repoPath, csvData, metadataData, false) + require.NoError(t, err) + + // Check .checkmarx folder was created + checkmarxDir := filepath.Join(repoPath, CheckmarxFolderName) + assert.True(t, fileExists(checkmarxDir), "should create .checkmarx directory") + + // CSV should NOT exist for public repos + csvPath := filepath.Join(checkmarxDir, ContributorsFileName) + assert.False(t, fileExists(csvPath), "should NOT create contributors.csv for public repo") + + // Metadata should exist + metadataPath := filepath.Join(checkmarxDir, MetadataFileName) + assert.True(t, fileExists(metadataPath), "should create metadata.json for all repos") +} + +func TestGenerateViaSystemGit_WithSystemGitCommands(t *testing.T) { + // Create real git repository using system git + repoPath := t.TempDir() + + // Initialize git repo + cmd := exec.Command("git", "init") + cmd.Dir = repoPath + require.NoError(t, cmd.Run()) + + // Configure git user + cmd = exec.Command("git", "config", "user.email", "test@example.com") + cmd.Dir = repoPath + require.NoError(t, cmd.Run()) + + cmd = exec.Command("git", "config", "user.name", "Test User") + cmd.Dir = repoPath + require.NoError(t, cmd.Run()) + + // Set remote + cmd = exec.Command("git", "remote", "add", "origin", "https://github.com/test/repo.git") + cmd.Dir = repoPath + require.NoError(t, cmd.Run()) + + // Create initial commit + testFile := filepath.Join(repoPath, "test.txt") + require.NoError(t, os.WriteFile(testFile, []byte("test content"), 0644)) + + cmd = exec.Command("git", "add", "test.txt") + cmd.Dir = repoPath + require.NoError(t, cmd.Run()) + + cmd = exec.Command("git", "commit", "-m", "initial commit") + cmd.Dir = repoPath + require.NoError(t, cmd.Run()) + + // Test generateViaSystemGit for private repo + err := generateViaSystemGit(repoPath, true) + require.NoError(t, err) + + // Verify files were created + checkmarxDir := filepath.Join(repoPath, CheckmarxFolderName) + assert.True(t, fileExists(checkmarxDir), "should create .checkmarx directory") + + csvPath := filepath.Join(checkmarxDir, ContributorsFileName) + assert.True(t, fileExists(csvPath), "should create contributors.csv for private repo") + + metadataPath := filepath.Join(checkmarxDir, MetadataFileName) + assert.True(t, fileExists(metadataPath), "should create metadata.json") + + // Verify metadata content + metadataBytes, err := os.ReadFile(metadataPath) + require.NoError(t, err) + + var metadata contributorsMetadata + require.NoError(t, json.Unmarshal(metadataBytes, &metadata)) + assert.Equal(t, "https://github.com/test/repo.git", metadata.RepositoryURL) + assert.NotEmpty(t, metadata.LastCommitHash) + assert.NotEmpty(t, metadata.LastCommitDate) + assert.GreaterOrEqual(t, metadata.CommitsCount, 1) +} + +func TestGenerateViaSystemGit_PublicRepo(t *testing.T) { + // Create real git repository using system git + repoPath := t.TempDir() + + // Initialize git repo + cmd := exec.Command("git", "init") + cmd.Dir = repoPath + require.NoError(t, cmd.Run()) + + // Configure git user + cmd = exec.Command("git", "config", "user.email", "test@example.com") + cmd.Dir = repoPath + require.NoError(t, cmd.Run()) + + cmd = exec.Command("git", "config", "user.name", "Test User") + cmd.Dir = repoPath + require.NoError(t, cmd.Run()) + + // Set remote + cmd = exec.Command("git", "remote", "add", "origin", "https://github.com/test/repo.git") + cmd.Dir = repoPath + require.NoError(t, cmd.Run()) + + // Create initial commit + testFile := filepath.Join(repoPath, "test.txt") + require.NoError(t, os.WriteFile(testFile, []byte("test content"), 0644)) + + cmd = exec.Command("git", "add", "test.txt") + cmd.Dir = repoPath + require.NoError(t, cmd.Run()) + + cmd = exec.Command("git", "commit", "-m", "initial commit") + cmd.Dir = repoPath + require.NoError(t, cmd.Run()) + + // Test generateViaSystemGit for PUBLIC repo + err := generateViaSystemGit(repoPath, false) + require.NoError(t, err) + + // Verify only metadata was created (no CSV for public repos) + checkmarxDir := filepath.Join(repoPath, CheckmarxFolderName) + assert.True(t, fileExists(checkmarxDir), "should create .checkmarx directory") + + csvPath := filepath.Join(checkmarxDir, ContributorsFileName) + assert.False(t, fileExists(csvPath), "should NOT create contributors.csv for public repo") + + metadataPath := filepath.Join(checkmarxDir, MetadataFileName) + assert.True(t, fileExists(metadataPath), "should create metadata.json") +} + +func TestWriteGeneratedFilesConditional_InvalidPathError(t *testing.T) { + // Test with invalid path to trigger MkdirAll error + invalidPath := "\x00invalid" + csvData := []byte("test csv") + metadataData := []byte("test metadata") + + err := writeGeneratedFilesConditional(invalidPath, csvData, metadataData, true) + assert.Error(t, err, "should return error for invalid path") +} + +func TestWriteGeneratedFilesConditional_FileExistsAsDirectory(t *testing.T) { + // Test error when .checkmarx path exists as file, not directory + repoPath := t.TempDir() + checkmarxDir := filepath.Join(repoPath, CheckmarxFolderName) + require.NoError(t, os.WriteFile(checkmarxDir, []byte("blocking"), 0644)) + + csvData := []byte("test csv") + metadataData := []byte("test metadata") + + err := writeGeneratedFilesConditional(repoPath, csvData, metadataData, true) + assert.Error(t, err, "should error when .checkmarx is a file") +} + +func TestGenerateAndWrite_WithCommits(t *testing.T) { + t.Run("creates metadata with commit count", func(t *testing.T) { + repoPath := t.TempDir() + repo, err := gogit.PlainInit(repoPath, false) + require.NoError(t, err) + + _, err = repo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{"https://github.com/test/repo.git"}, + }) + require.NoError(t, err) + + worktree, err := repo.Worktree() + require.NoError(t, err) + + sig := &object.Signature{Name: "User", Email: "user@example.com", When: time.Now()} + _, err = worktree.Commit("commit1", &gogit.CommitOptions{ + Author: sig, + AllowEmptyCommits: true, + }) + require.NoError(t, err) + + err = GenerateAndWrite(repoPath, true) + assert.NoError(t, err) + + metadataPath := filepath.Join(repoPath, CheckmarxFolderName, MetadataFileName) + data, err := os.ReadFile(metadataPath) + require.NoError(t, err) + + var metadata contributorsMetadata + err = json.Unmarshal(data, &metadata) + require.NoError(t, err) + assert.Equal(t, 1, metadata.CommitsCount) + }) + + t.Run("creates files for private repo with commit", func(t *testing.T) { + repoPath := t.TempDir() + repo, err := gogit.PlainInit(repoPath, false) + require.NoError(t, err) + + _, err = repo.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{"https://github.com/private/repo.git"}, + }) + require.NoError(t, err) + + worktree, err := repo.Worktree() + require.NoError(t, err) + + sig := &object.Signature{Name: "Dev", Email: "dev@example.com", When: time.Now()} + _, err = worktree.Commit("work", &gogit.CommitOptions{ + Author: sig, + AllowEmptyCommits: true, + }) + require.NoError(t, err) + + err = GenerateAndWrite(repoPath, true) + assert.NoError(t, err) + + metadataPath := filepath.Join(repoPath, CheckmarxFolderName, MetadataFileName) + assert.True(t, fileExists(metadataPath)) + }) +} + +func TestCommitsSince_FiltersOldCommits(t *testing.T) { + now := time.Now() + repoPath := newGitMetadataTestRepo(t, "https://example.com/org/repo.git", []testCommit{ + {email: "old@example.com", name: "Old", when: now.Add(-200 * 24 * time.Hour)}, + {email: "recent@example.com", name: "Recent", when: now.Add(-1 * 24 * time.Hour)}, + }) + + repo, _ := gogit.PlainOpen(repoPath) + since := now.Add(-100 * 24 * time.Hour) + commits, _ := commitsSince(repo, since) + + assert.Len(t, commits, 1) + assert.Equal(t, "recent@example.com", commits[0].Author.Email) +} + +func TestWriteGeneratedFilesConditional_CSVWriteError(t *testing.T) { + repoPath := t.TempDir() + checkmarxDir := filepath.Join(repoPath, CheckmarxFolderName) + + // Create a file at the path where we'd create the directory, causing MkdirAll to fail + err := os.WriteFile(checkmarxDir, []byte("blocking file"), 0o600) + require.NoError(t, err) + + err = writeGeneratedFilesConditional(repoPath, []byte("csv"), []byte(`{}`), true) + assert.Error(t, err) + assert.Contains(t, err.Error(), "could not create .checkmarx directory") +} + +func TestBuildContributorsCSV_MultipleEmailsPreservesOrder(t *testing.T) { + now := time.Now() + repoPath := newGitMetadataTestRepo(t, "", []testCommit{ + {email: "user1@example.com", name: "User One", when: now}, + {email: "user2@example.com", name: "User Two", when: now.Add(-1 * time.Hour)}, + {email: "user3@example.com", name: "User Three", when: now.Add(-48 * time.Hour)}, + }) + + require.NoError(t, GenerateAndWrite(repoPath, true)) + + csvContent, _ := readGitMetadataFiles(t, repoPath) + reader := csv.NewReader(strings.NewReader(csvContent)) + records, err := reader.ReadAll() + require.NoError(t, err) + + // Should have 3 unique emails + assert.Len(t, records, 3) + // Each record should have 4 fields: date, hash, email, name + for _, record := range records { + assert.Len(t, record, 4) + // First field should be parseable as RFC3339 + _, err := time.Parse(time.RFC3339, record[0]) + assert.NoError(t, err) + // Email should be in field 2 + assert.Contains(t, record[2], "@example.com") + } +} + +func TestDetectRepositoryPrivacy_FallbackToSystemGit(t *testing.T) { + repoPath := newGitMetadataTestRepo(t, "https://example.com/org/repo.git", []testCommit{ + {email: "alice@example.com", name: "Alice", when: time.Now()}, + }) + + mockClient := &http.Client{Timeout: 5 * time.Second} + + // When detectRepositoryPrivacy is called, it should try go-git first (succeeds), then isPrivateByURL; since example.com is not a real SCM, the function should handle it gracefully and default to private=true. + result := detectRepositoryPrivacy(repoPath, mockClient) + assert.True(t, result, "should conservatively default to private for unknown hosts") +} + +func TestGenerateAndWrite_WithCSVSizeLimit(t *testing.T) { + now := time.Now() + // Create repo with many contributors to potentially trigger size warning + var commits []testCommit + for i := 0; i < 50; i++ { + commits = append(commits, testCommit{ + email: fmt.Sprintf("user%d@example.com", i), + name: fmt.Sprintf("User %d", i), + when: now.Add(-time.Duration(i) * time.Hour), + }) + } + repoPath := newGitMetadataTestRepo(t, "https://example.com/org/repo.git", commits) + + err := GenerateAndWrite(repoPath, true) + assert.NoError(t, err, "should handle repos with many contributors") + + csvContent, metadata := readGitMetadataFiles(t, repoPath) + lines := strings.Split(strings.TrimRight(csvContent, "\n"), "\n") + assert.Equal(t, 50, len(lines), "should generate CSV with all unique contributors") + assert.Equal(t, 50, metadata.CommitsCount) +} + +func TestDetectRepositoryPrivacy_ExistingCSVFile(t *testing.T) { + repoPath := t.TempDir() + checkmarxDir := filepath.Join(repoPath, CheckmarxFolderName) + csvPath := filepath.Join(checkmarxDir, ContributorsFileName) + + // Create CSV file to indicate previous private repo + require.NoError(t, os.MkdirAll(checkmarxDir, 0o755)) + require.NoError(t, os.WriteFile(csvPath, []byte("test"), 0o644)) + + mockClient := &http.Client{Timeout: 5 * time.Second} + result := detectRepositoryPrivacy(repoPath, mockClient) + assert.True(t, result, "should return true when CSV file exists (repo was private)") +} + +func TestDetectRepositoryPrivacy_NoRemoteURL(t *testing.T) { + repoPath := t.TempDir() + // Create an empty git repo with no remote + _, err := gogit.PlainInit(repoPath, false) + require.NoError(t, err) + + // Don't add any remote + mockClient := &http.Client{Timeout: 5 * time.Second} + result := detectRepositoryPrivacy(repoPath, mockClient) + // Should conservatively default to private when no remote found + assert.True(t, result) +} diff --git a/internal/commands/root.go b/internal/commands/root.go index 74602d61d..3b968210e 100644 --- a/internal/commands/root.go +++ b/internal/commands/root.go @@ -133,6 +133,9 @@ func NewAstCLI( return err } PrintConfiguration() + if viper.GetBool(params.InsecureFlag) { + fmt.Println("WARNING: --insecure flag is enabled. This disables SSL/TLS certificate verification. Do NOT use this flag in production unless your security team has explicitly evaluated and approved the risk.") + } err = configuration.LoadConfiguration() if err != nil { return err diff --git a/internal/commands/scan.go b/internal/commands/scan.go index c109566c9..ad0cbebb0 100644 --- a/internal/commands/scan.go +++ b/internal/commands/scan.go @@ -62,6 +62,7 @@ const ( containerImagesFlagError = "--container-images flag error" git = "git" + gitFolderName = ".git" invalidSSHSource = "provided source does not need a key. Make sure you are defining the right source or remove the flag --ssh-key" errorUnzippingFile = "an error occurred while unzipping file. Reason: " containerRun = "run" @@ -933,6 +934,7 @@ func scanCreateSubCommand( createScanCmd.PersistentFlags().Bool(commonParams.GitIgnoreFileFilterFlag, false, commonParams.GitIgnoreFileFilterUsage) createScanCmd.PersistentFlags().StringSlice(commonParams.AntFilterFlag, []string{}, commonParams.AntFilterUsage) createScanCmd.PersistentFlags().Bool(commonParams.SkipDefaultFilterFlag, false, commonParams.SkipDefaultFilterFlagUsage) + createScanCmd.PersistentFlags().Bool(commonParams.ExcludeGitFolderFlag, false, commonParams.ExcludeGitFolderFlagUsage) return createScanCmd } @@ -1649,7 +1651,7 @@ func scanTypeEnabled(scanType string) bool { return false } -func compressFolder(sourceDir, filter, userIncludeFilter, scaResolver string, antMatcher filtering.Matcher, skipDefaultFilter bool) (string, error) { +func compressFolder(sourceDir, filter, userIncludeFilter, scaResolver string, antMatcher filtering.Matcher, skipDefaultFilter, includeGeneratedCsvJson, excludeGitFolder bool) (string, error) { scaToolPath := scaResolver outputFile, err := os.CreateTemp(os.TempDir(), "cx-*.zip") if err != nil { @@ -1677,7 +1679,7 @@ func compressFolder(sourceDir, filter, userIncludeFilter, scaResolver string, an } } else { // Add directory files normally - err = addDirFiles(zipWriter, "", sourceDir, getExcludeFilters(filter, skipDefaultFilter), getIncludeFilters(userIncludeFilter, skipDefaultFilter), antMatcher) + err = addDirFiles(zipWriter, "", sourceDir, getExcludeFilters(filter, skipDefaultFilter), getIncludeFilters(userIncludeFilter, skipDefaultFilter), antMatcher, excludeGitFolder) if err != nil { return "", err } @@ -1690,6 +1692,13 @@ func compressFolder(sourceDir, filter, userIncludeFilter, scaResolver string, an } } + // Add contributors.csv/metadata.json only if they were just freshly generated without error. + if includeGeneratedCsvJson { + if err := addGeneratedContributorsFiles(zipWriter, sourceDir); err != nil { + return "", err + } + } + // Close the file err = zipWriter.Close() if err != nil { @@ -1808,7 +1817,7 @@ func addDirFilesIgnoreFilter(zipWriter *zip.Writer, baseDir, parentDir string) e return nil } -func addDirFiles(zipWriter *zip.Writer, baseDir, parentDir string, filters, includeFilters []string, antMatcher filtering.Matcher) error { +func addDirFiles(zipWriter *zip.Writer, baseDir, parentDir string, filters, includeFilters []string, antMatcher filtering.Matcher, excludeGitFolder bool) error { fileEntries, err := os.ReadDir(parentDir) if err != nil { return err @@ -1821,7 +1830,7 @@ func addDirFiles(zipWriter *zip.Writer, baseDir, parentDir string, filters, incl } if util.IsDirOrSymLinkToDir(parentDir, fileInfo) { - err = handleDir(zipWriter, baseDir, parentDir, filters, includeFilters, fileInfo, antMatcher) + err = handleDir(zipWriter, baseDir, parentDir, filters, includeFilters, fileInfo, antMatcher, excludeGitFolder) } else { err = handleFile(zipWriter, baseDir, parentDir, filters, includeFilters, fileInfo, antMatcher) } @@ -1852,6 +1861,11 @@ func handleFile( } // relPath is forward-slash path from source root, used by antMatcher. relPath := filepath.ToSlash(baseDir + file.Name()) + // Exclude .checkmarx files from normal walk; re-add only if successfully generated in compressFolder + if isGeneratedContributorsFile(relPath) { + logger.PrintIfVerbose("Excluded (added separately): " + fileName) + return nil + } if filterMatched(includeFilters, file.Name()) && filterMatched(filters, file.Name()) && !antMatcher.Excluded(relPath, false) { logger.PrintIfVerbose("Included: " + fileName) dat, err := ioutil.ReadFile(parentDir + file.Name()) @@ -1884,9 +1898,15 @@ func handleDir( includeFilters []string, file fs.FileInfo, antMatcher filtering.Matcher, + excludeGitFolder bool, ) error { // Check if folder belongs to the disabled exclusions if commonParams.DisabledExclusions[file.Name()] { + // Exclude .git folder when --exclude-git-folder flag is passed + if excludeGitFolder && file.Name() == gitFolderName { + logger.PrintIfVerbose("The folder " + file.Name() + " is being excluded (--exclude-git-folder flag passed)") + return nil + } logger.PrintIfVerbose("The folder " + file.Name() + " is being included") newParent, newBase := GetNewParentAndBase(parentDir, file, baseDir) return addDirFilesIgnoreFilter(zipWriter, newBase, newParent) @@ -1917,7 +1937,7 @@ func handleDir( } newParent, newBase := GetNewParentAndBase(parentDir, file, baseDir) - return addDirFiles(zipWriter, newBase, newParent, filters, includeFilters, antMatcher) + return addDirFiles(zipWriter, newBase, newParent, filters, includeFilters, antMatcher, excludeGitFolder) } func isDirFiltered(filename string, filters []string) (bool, error) { @@ -2144,6 +2164,8 @@ func getUploadURLFromSource(cmd *cobra.Command, uploadsWrapper wrappers.UploadsW scaResolverParams, scaResolver := getScaResolverFlags(cmd) isSbom, _ := cmd.PersistentFlags().GetBool(commonParams.SbomFlag) isGitIgnoreFilter, _ := cmd.Flags().GetBool(commonParams.GitIgnoreFileFilterFlag) + excludeGitFolder, _ := cmd.Flags().GetBool(commonParams.ExcludeGitFolderFlag) + contributorsCsvFlag, _ := wrappers.GetSpecificFeatureFlag(featureFlagsWrapper, wrappers.RepostoreCustomerContributorsCsvEnabled) // Build the Ant-style matcher from --file-filter-ext patterns. // Construction errors are surfaced immediately so the user gets clear @@ -2204,6 +2226,7 @@ func getUploadURLFromSource(cmd *cobra.Command, uploadsWrapper wrappers.UploadsW var errorUnzippingFile error userProvidedZip := len(zipFilePath) > 0 + contributorsCsvEnabled := (contributorsCsvFlag != nil && contributorsCsvFlag.Status) && !userProvidedZip // containerScanTriggered must stay in this condition: without it, a container scan // run with --containers-local-resolution and --skip-default-filter (and no @@ -2303,7 +2326,23 @@ func getUploadURLFromSource(cmd *cobra.Command, uploadsWrapper wrappers.UploadsW } } else { if !isSbom { - zipFilePath, dirPathErr = compressFolder(directoryPath, sourceDirFilter, userIncludeFilter, scaResolver, antMatcher, skipDefaultFilter) + // True only if contributors.csv/metadata.json were just generated successfully without error. + includeGeneratedCsvJson := false + if contributorsCsvEnabled { + httpClient := wrappers.GetClient(privacyDetectionTimeout) + isPrivate := detectRepositoryPrivacy(directoryPath, httpClient) + if genErr := GenerateAndWrite(directoryPath, isPrivate); genErr != nil { + logger.PrintIfVerbose("Skipping contributors.csv/metadata.json generation: " + genErr.Error()) + } else { + includeGeneratedCsvJson = true + } + } + zipFilePath, dirPathErr = compressFolder(directoryPath, sourceDirFilter, userIncludeFilter, scaResolver, antMatcher, skipDefaultFilter, includeGeneratedCsvJson, excludeGitFolder) + + // Clean up generated contributors files after successful zip creation + if dirPathErr == nil && includeGeneratedCsvJson { + cleanGeneratedContributorsFiles(directoryPath) + } } // Clean up .checkmarx/containers directory after successful mixed scan (including containers) compression @@ -4373,7 +4412,7 @@ func hasGitRepository(source string) bool { } // Check if .git exists in the root directory - gitPath := filepath.Join(sourceTrimmed, ".git") + gitPath := filepath.Join(sourceTrimmed, gitFolderName) if _, err := os.Stat(gitPath); err == nil { return true } @@ -4389,7 +4428,7 @@ func searchGitInSubdirectories(sourcePath string) bool { if err != nil || found { return nil } - if info.IsDir() && info.Name() == ".git" { + if info.IsDir() && info.Name() == gitFolderName { found = true return filepath.SkipAll } @@ -4551,3 +4590,83 @@ func readGitIgnoreFromZip(zipPath string) ([]byte, error) { } return []byte(""), fmt.Errorf(".gitignore not found in zip: %s", zipPath) } + +// isGeneratedContributorsFile checks if file is contributors.csv or metadata.json to skip in zip walk. +func isGeneratedContributorsFile(relPath string) bool { + return relPath == CheckmarxFolderName+"/"+ContributorsFileName || + relPath == CheckmarxFolderName+"/"+MetadataFileName +} + +// addGeneratedContributorsFiles reads and writes contributors.csv/metadata.json to zip; missing files OK. +func addGeneratedContributorsFiles(zipWriter *zip.Writer, sourceDir string) error { + for _, fileName := range []string{ContributorsFileName, MetadataFileName} { + filePath := filepath.Join(sourceDir, CheckmarxFolderName, fileName) + dat, err := os.ReadFile(filePath) + if err != nil { + if os.IsNotExist(err) { + logger.PrintIfVerbose("Skipping " + fileName + ": not found under " + CheckmarxFolderName + "/") + continue + } + return err + } + + zipEntryName := CheckmarxFolderName + "/" + fileName + f, err := zipWriter.Create(zipEntryName) + if err != nil { + return err + } + if _, err := f.Write(dat); err != nil { + return err + } + logger.PrintIfVerbose("Included: " + zipEntryName) + } + return nil +} + +// cleanGeneratedContributorsFiles removes contributors.csv and metadata.json after zip creation. +// Removes both files if present (either or both may exist). Preserves .checkmarx folder if other files remain. +// Only deletes .checkmarx folder if it becomes completely empty after both files are removed. +func cleanGeneratedContributorsFiles(directoryPath string) { + checkmarxDir := filepath.Join(directoryPath, ".checkmarx") + if _, err := os.Stat(checkmarxDir); os.IsNotExist(err) { + return + } + + csvPath := filepath.Join(checkmarxDir, "contributors.csv") + jsonPath := filepath.Join(checkmarxDir, "metadata.json") + fileRemoved := false + + // Attempt to remove contributors.csv if it exists + if _, err := os.Stat(csvPath); err == nil { + if rmErr := os.Remove(csvPath); rmErr != nil { + logger.PrintIfVerbose(fmt.Sprintf("Warning: Failed to remove contributors.csv: %s", rmErr.Error())) + } else { + logger.PrintIfVerbose("Removed contributors.csv after zip creation") + fileRemoved = true + } + } + + // Attempt to remove metadata.json if it exists + if _, err := os.Stat(jsonPath); err == nil { + if rmErr := os.Remove(jsonPath); rmErr != nil { + logger.PrintIfVerbose(fmt.Sprintf("Warning: Failed to remove metadata.json: %s", rmErr.Error())) + } else { + logger.PrintIfVerbose("Removed metadata.json after zip creation") + fileRemoved = true + } + } + + // Only remove .checkmarx folder if it's empty after both contributor files removed (either or both may have existed) + if fileRemoved { + entries, err := os.ReadDir(checkmarxDir) + if err == nil && len(entries) == 0 { + if rmErr := os.Remove(checkmarxDir); rmErr != nil { + logger.PrintIfVerbose(fmt.Sprintf("Warning: Failed to remove empty .checkmarx directory: %s", rmErr.Error())) + return + } + logger.PrintIfVerbose("Removed empty .checkmarx directory (empty after removing contributor files)") + } else if err == nil && len(entries) > 0 { + logger.PrintIfVerbose(fmt.Sprintf("Kept .checkmarx directory (contains %d other file(s) e.g., containers/)", len(entries))) + } + } +} diff --git a/internal/commands/scan_test.go b/internal/commands/scan_test.go index 936a42398..374bd3c9b 100644 --- a/internal/commands/scan_test.go +++ b/internal/commands/scan_test.go @@ -5348,7 +5348,7 @@ func TestSbomFileExcludedFromZip_WithCustomOutputName(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5379,7 +5379,7 @@ func TestDefaultSbomFileAlwaysExcludedFromZip(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5412,7 +5412,7 @@ func TestSbomFileExcludedFromZip_InSubdirectory(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5451,7 +5451,7 @@ func TestSbomFileExcludedFromZip_AbsoluteSubdirWithCustomName(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5536,7 +5536,7 @@ func TestCompressFolder_DefaultBehaviorUnchanged(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5557,7 +5557,7 @@ func TestCompressFolder_SkipDefaultFilter(t *testing.T) { noopMatcher, matcherErr := filtering.NewAntMatcher(nil) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, true) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, true, false, false) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5587,7 +5587,7 @@ func TestCompressFolder_SkipDefaultFilter_WithAntFilterExclude(t *testing.T) { antMatcher, matcherErr := filtering.NewAntMatcher([]string{"!excluded_by_ant/**"}) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", antMatcher, true) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", antMatcher, true, false, false) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5608,7 +5608,7 @@ func TestCompressFolder_SkipDefaultFilter_WithAntFilterIncludeOnly(t *testing.T) antMatcher, matcherErr := filtering.NewAntMatcher([]string{"**/*.customext"}) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", antMatcher, true) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", antMatcher, true, false, false) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -5634,7 +5634,7 @@ func TestCompressFolder_DefaultFilters_WithAntFilter(t *testing.T) { antMatcher, matcherErr := filtering.NewAntMatcher([]string{"!keep_dir/**"}) assert.NilError(t, matcherErr) - zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", antMatcher, false) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", antMatcher, false, false, false) assert.NilError(t, err) defer func() { _ = os.Remove(zipPath) }() @@ -6029,3 +6029,303 @@ func TestWasOrWere(t *testing.T) { assert.Equal(t, wasOrWere(2), "were") assert.Equal(t, wasOrWere(0), "were") } + +// zipFileCount returns count of entries in zip with given filename to guard against duplicates. +func zipFileCount(t *testing.T, zipPath, filename string) int { + t.Helper() + r, err := zip.OpenReader(zipPath) + assert.NilError(t, err) + defer func() { _ = r.Close() }() + count := 0 + for _, f := range r.File { + if filepath.Base(f.Name) == filename || f.Name == filename { + count++ + } + } + return count +} + +func TestCompressFolder_GitExcluded_WhenContributorsCsvEnabled(t *testing.T) { + projectDir, err := os.MkdirTemp("", "contributors-csv-git-exclude-*") + assert.NilError(t, err) + defer func() { _ = os.RemoveAll(projectDir) }() + + assert.NilError(t, os.WriteFile(filepath.Join(projectDir, "main.go"), []byte("package main"), 0o600)) + gitDir := filepath.Join(projectDir, ".git") + assert.NilError(t, os.MkdirAll(gitDir, 0o700)) + assert.NilError(t, os.WriteFile(filepath.Join(gitDir, "HEAD"), []byte("ref: refs/heads/main"), 0o600)) + + noopMatcher, matcherErr := filtering.NewAntMatcher(nil) + assert.NilError(t, matcherErr) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, true, true) + assert.NilError(t, err) + defer func() { _ = os.Remove(zipPath) }() + + assert.Equal(t, true, zipContainsFile(t, zipPath, "main.go")) + assert.Equal(t, false, zipContainsFile(t, zipPath, "HEAD"), + ".git contents should be excluded when --exclude-git-folder flag is passed") +} + +func TestCompressFolder_GitIncluded_WhenContributorsCsvDisabled(t *testing.T) { + projectDir, err := os.MkdirTemp("", "contributors-csv-git-include-*") + assert.NilError(t, err) + defer func() { _ = os.RemoveAll(projectDir) }() + + assert.NilError(t, os.WriteFile(filepath.Join(projectDir, "main.go"), []byte("package main"), 0o600)) + gitDir := filepath.Join(projectDir, ".git") + assert.NilError(t, os.MkdirAll(gitDir, 0o700)) + assert.NilError(t, os.WriteFile(filepath.Join(gitDir, "HEAD"), []byte("ref: refs/heads/main"), 0o600)) + + noopMatcher, matcherErr := filtering.NewAntMatcher(nil) + assert.NilError(t, matcherErr) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) + assert.NilError(t, err) + defer func() { _ = os.Remove(zipPath) }() + + assert.Equal(t, true, zipContainsFile(t, zipPath, "main.go")) + assert.Equal(t, true, zipContainsFile(t, zipPath, "HEAD"), + ".git contents must still be force-included when --exclude-git-folder flag is not passed") +} + +func TestCompressFolder_ContributorsFilesForceIncluded_WhenEnabled(t *testing.T) { + projectDir, err := os.MkdirTemp("", "contributors-csv-force-include-*") + assert.NilError(t, err) + defer func() { _ = os.RemoveAll(projectDir) }() + + assert.NilError(t, os.WriteFile(filepath.Join(projectDir, "main.go"), []byte("package main"), 0o600)) + checkmarxDir := filepath.Join(projectDir, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0o700)) + assert.NilError(t, os.WriteFile(filepath.Join(checkmarxDir, "contributors.csv"), + []byte("2025-09-30T10:35:05+03:00,abc123,alice@example.com,Alice\n"), 0o600)) + assert.NilError(t, os.WriteFile(filepath.Join(checkmarxDir, "metadata.json"), + []byte(`{"repositoryUrl":"https://example.com/repo.git"}`), 0o600)) + + noopMatcher, matcherErr := filtering.NewAntMatcher(nil) + assert.NilError(t, matcherErr) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, true, false) + assert.NilError(t, err) + defer func() { _ = os.Remove(zipPath) }() + + assert.Equal(t, true, zipContainsFile(t, zipPath, "main.go")) + assert.Equal(t, 1, zipFileCount(t, zipPath, "contributors.csv"), + "contributors.csv must be present exactly once, despite *.csv not being in the default include-filter allowlist") + assert.Equal(t, 1, zipFileCount(t, zipPath, "metadata.json"), + "metadata.json must be present exactly once (not duplicated by both the normal walk and the explicit add-back step)") +} + +func TestCompressFolder_ContributorsFilesNotIncluded_WhenDisabled(t *testing.T) { + projectDir, err := os.MkdirTemp("", "contributors-csv-disabled-*") + assert.NilError(t, err) + defer func() { _ = os.RemoveAll(projectDir) }() + + assert.NilError(t, os.WriteFile(filepath.Join(projectDir, "main.go"), []byte("package main"), 0o600)) + checkmarxDir := filepath.Join(projectDir, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0o700)) + assert.NilError(t, os.WriteFile(filepath.Join(checkmarxDir, "contributors.csv"), + []byte("2025-09-30T10:35:05+03:00,abc123,alice@example.com,Alice\n"), 0o600)) + + noopMatcher, matcherErr := filtering.NewAntMatcher(nil) + assert.NilError(t, matcherErr) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, false) + assert.NilError(t, err) + defer func() { _ = os.Remove(zipPath) }() + + assert.Equal(t, true, zipContainsFile(t, zipPath, "main.go")) + assert.Equal(t, false, zipContainsFile(t, zipPath, "contributors.csv"), + "without the feature flag, contributors.csv should be dropped by the default include-filter allowlist, same as before this feature existed") +} + +func TestCompressFolder_StaleFilesNotIncluded_WhenGenerationFailedThisRun(t *testing.T) { + projectDir, err := os.MkdirTemp("", "contributors-csv-stale-*") + assert.NilError(t, err) + defer func() { _ = os.RemoveAll(projectDir) }() + + assert.NilError(t, os.WriteFile(filepath.Join(projectDir, "main.go"), []byte("package main"), 0o600)) + gitDir := filepath.Join(projectDir, ".git") + assert.NilError(t, os.MkdirAll(gitDir, 0o700)) + assert.NilError(t, os.WriteFile(filepath.Join(gitDir, "HEAD"), []byte("ref: refs/heads/main"), 0o600)) + + checkmarxDir := filepath.Join(projectDir, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0o700)) + assert.NilError(t, os.WriteFile(filepath.Join(checkmarxDir, "contributors.csv"), + []byte("2025-09-30T10:35:05+03:00,abc123,alice@example.com,Alice\n"), 0o600)) + assert.NilError(t, os.WriteFile(filepath.Join(checkmarxDir, "metadata.json"), + []byte(`{"repositoryUrl":"https://example.com/repo.git"}`), 0o600)) + + noopMatcher, matcherErr := filtering.NewAntMatcher(nil) + assert.NilError(t, matcherErr) + zipPath, err := compressFolder(sbomTestSourceDir(projectDir), "", "", "", noopMatcher, false, false, true) + assert.NilError(t, err) + defer func() { _ = os.Remove(zipPath) }() + + assert.Equal(t, true, zipContainsFile(t, zipPath, "main.go")) + assert.Equal(t, false, zipContainsFile(t, zipPath, "HEAD"), + ".git should still be excluded - that decision is tied to the flag alone, not generation success") + assert.Equal(t, false, zipContainsFile(t, zipPath, "contributors.csv"), + "stale contributors.csv from a previous run must not be picked up when this run's generation failed") + assert.Equal(t, false, zipContainsFile(t, zipPath, "metadata.json"), + "stale metadata.json from a previous run must not be picked up when this run's generation failed") +} + +func TestIsGeneratedContributorsFile(t *testing.T) { + tests := []struct { + relPath string + expected bool + desc string + }{ + {".checkmarx/contributors.csv", true, "exact match for CSV file"}, + {".checkmarx/metadata.json", true, "exact match for JSON file"}, + {".checkmarx/other.txt", false, "non-generated file in .checkmarx"}, + {"contributors.csv", false, "CSV file not in .checkmarx"}, + {"metadata.json", false, "JSON file not in .checkmarx"}, + {".checkmarx/", false, "directory path"}, + {"", false, "empty path"}, + {".checkmarx/contributors.csv/", false, "trailing slash"}, + {"nested/.checkmarx/contributors.csv", false, "nested .checkmarx path"}, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + result := isGeneratedContributorsFile(tt.relPath) + assert.Equal(t, tt.expected, result, "path: %s", tt.relPath) + }) + } +} + +func TestAddGeneratedContributorsFiles(t *testing.T) { + t.Run("both files exist and are added to zip", func(t *testing.T) { + sourceDir := t.TempDir() + checkmarxDir := filepath.Join(sourceDir, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0o700)) + + csvContent := []byte("2025-09-30T10:35:05Z,abc123,alice@example.com,Alice\n") + jsonContent := []byte(`{"repositoryUrl":"https://example.com/repo.git","commitsCount":5}`) + + assert.NilError(t, os.WriteFile(filepath.Join(checkmarxDir, "contributors.csv"), csvContent, 0o600)) + assert.NilError(t, os.WriteFile(filepath.Join(checkmarxDir, "metadata.json"), jsonContent, 0o600)) + + zipPath := filepath.Join(t.TempDir(), "test.zip") + zipFile, err := os.Create(zipPath) + assert.NilError(t, err) + defer func() { _ = zipFile.Close() }() + + zipWriter := zip.NewWriter(zipFile) + defer func() { _ = zipWriter.Close() }() + + err = addGeneratedContributorsFiles(zipWriter, sourceDir) + assert.NilError(t, err) + assert.NilError(t, zipWriter.Close()) + + assert.Equal(t, true, zipContainsFile(t, zipPath, ".checkmarx/contributors.csv")) + assert.Equal(t, true, zipContainsFile(t, zipPath, ".checkmarx/metadata.json")) + }) + + t.Run("only CSV file exists", func(t *testing.T) { + sourceDir := t.TempDir() + checkmarxDir := filepath.Join(sourceDir, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0o700)) + + csvContent := []byte("2025-09-30T10:35:05Z,abc123,alice@example.com,Alice\n") + assert.NilError(t, os.WriteFile(filepath.Join(checkmarxDir, "contributors.csv"), csvContent, 0o600)) + + zipPath := filepath.Join(t.TempDir(), "test.zip") + zipFile, err := os.Create(zipPath) + assert.NilError(t, err) + defer func() { _ = zipFile.Close() }() + + zipWriter := zip.NewWriter(zipFile) + defer func() { _ = zipWriter.Close() }() + + err = addGeneratedContributorsFiles(zipWriter, sourceDir) + assert.NilError(t, err) + assert.NilError(t, zipWriter.Close()) + + assert.Equal(t, true, zipContainsFile(t, zipPath, ".checkmarx/contributors.csv")) + assert.Equal(t, false, zipContainsFile(t, zipPath, ".checkmarx/metadata.json")) + }) + + t.Run("no files exist should not error", func(t *testing.T) { + sourceDir := t.TempDir() + checkmarxDir := filepath.Join(sourceDir, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0o700)) + + zipPath := filepath.Join(t.TempDir(), "test.zip") + zipFile, err := os.Create(zipPath) + assert.NilError(t, err) + defer func() { _ = zipFile.Close() }() + + zipWriter := zip.NewWriter(zipFile) + defer func() { _ = zipWriter.Close() }() + + err = addGeneratedContributorsFiles(zipWriter, sourceDir) + assert.NilError(t, err, "should not error when files don't exist") + }) +} + +func TestCleanGeneratedContributorsFiles(t *testing.T) { + t.Run("removes both files and empty .checkmarx folder", func(t *testing.T) { + dirPath := t.TempDir() + checkmarxDir := filepath.Join(dirPath, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0o700)) + + csvPath := filepath.Join(checkmarxDir, "contributors.csv") + jsonPath := filepath.Join(checkmarxDir, "metadata.json") + assert.NilError(t, os.WriteFile(csvPath, []byte("data"), 0o600)) + assert.NilError(t, os.WriteFile(jsonPath, []byte("data"), 0o600)) + + cleanGeneratedContributorsFiles(dirPath) + + assert.Equal(t, false, fileExists(csvPath), "CSV should be removed") + assert.Equal(t, false, fileExists(jsonPath), "JSON should be removed") + assert.Equal(t, false, fileExists(checkmarxDir), ".checkmarx should be removed when empty") + }) + + t.Run("preserves .checkmarx folder if other files exist", func(t *testing.T) { + dirPath := t.TempDir() + checkmarxDir := filepath.Join(dirPath, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0o700)) + + csvPath := filepath.Join(checkmarxDir, "contributors.csv") + jsonPath := filepath.Join(checkmarxDir, "metadata.json") + otherPath := filepath.Join(checkmarxDir, "other.txt") + + assert.NilError(t, os.WriteFile(csvPath, []byte("data"), 0o600)) + assert.NilError(t, os.WriteFile(jsonPath, []byte("data"), 0o600)) + assert.NilError(t, os.WriteFile(otherPath, []byte("data"), 0o600)) + + cleanGeneratedContributorsFiles(dirPath) + + assert.Equal(t, false, fileExists(csvPath), "CSV should be removed") + assert.Equal(t, false, fileExists(jsonPath), "JSON should be removed") + assert.Equal(t, true, fileExists(checkmarxDir), ".checkmarx should be preserved") + assert.Equal(t, true, fileExists(otherPath), "other files should be preserved") + }) + + t.Run("handles only CSV file existing", func(t *testing.T) { + dirPath := t.TempDir() + checkmarxDir := filepath.Join(dirPath, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0o700)) + + csvPath := filepath.Join(checkmarxDir, "contributors.csv") + assert.NilError(t, os.WriteFile(csvPath, []byte("data"), 0o600)) + + cleanGeneratedContributorsFiles(dirPath) + + assert.Equal(t, false, fileExists(csvPath), "CSV should be removed") + assert.Equal(t, false, fileExists(checkmarxDir), ".checkmarx should be removed when empty") + }) + + t.Run("handles no .checkmarx folder gracefully", func(t *testing.T) { + dirPath := t.TempDir() + cleanGeneratedContributorsFiles(dirPath) + }) + + t.Run("handles missing files gracefully", func(t *testing.T) { + dirPath := t.TempDir() + checkmarxDir := filepath.Join(dirPath, ".checkmarx") + assert.NilError(t, os.MkdirAll(checkmarxDir, 0o700)) + + cleanGeneratedContributorsFiles(dirPath) + assert.Equal(t, true, fileExists(checkmarxDir), ".checkmarx should still exist") + }) +} diff --git a/internal/params/flags.go b/internal/params/flags.go index 0dd7e1089..eae820f30 100644 --- a/internal/params/flags.go +++ b/internal/params/flags.go @@ -197,6 +197,8 @@ const ( LogFileConsoleUsage = "Saves logs to the specified file path as well as to the console" SkipDefaultFilterFlag = "skip-default-filter" SkipDefaultFilterFlagUsage = "Skip the default file filter." + ExcludeGitFolderFlag = "exclude-git-folder" + ExcludeGitFolderFlagUsage = "Exclude .git folder from scan source upload zip." GitIgnoreFileFilterFlag = "use-gitignore" GitIgnoreFileFilterUsage = "Exclude files and directories from the scan based on the patterns defined in the directory's .gitignore file" AntFilterFlag = "file-filter-ext" diff --git a/internal/wrappers/feature-flags.go b/internal/wrappers/feature-flags.go index ec0a31bd4..c4008f64e 100644 --- a/internal/wrappers/feature-flags.go +++ b/internal/wrappers/feature-flags.go @@ -22,6 +22,9 @@ const maxRetries = 3 const IncreaseFileUploadLimit = "INCREASE_FILE_UPLOAD_LIMIT" const ScaDeltaScanEnabled = "SCA_DELTASCAN_ENABLED" +// RepostoreCustomerContributorsCsvEnabled is the feature flag for generating contributors.csv and metadata.json. +const RepostoreCustomerContributorsCsvEnabled = "REPOSTORE_CUSTOMER_CONTRIBUTORS_CSV_ENABLED" + // AISupplyChainGAEnabled is the feature flag for AI Supply Chain Engine GA. const AISupplyChainGAEnabled = "AI_SUPPLY_CHAIN_ENGINE_GA_ENABLED" diff --git a/test/integration/exclude_git_test.go b/test/integration/exclude_git_test.go new file mode 100644 index 000000000..245a3d2aa --- /dev/null +++ b/test/integration/exclude_git_test.go @@ -0,0 +1,110 @@ +//go:build integration + +package integration + +import ( + "bytes" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/checkmarx/ast-cli/internal/params" + "gotest.tools/assert" +) + +// Integration tests for --exclude-git-folder flag +// Clones a public repo once at package level and reuses for all tests + +var ( + clonedRepoPath string + cloneOnce sync.Once + cloneErr error +) + +// clonePublicRepoOnce clones the repo exactly once and caches the path +func clonePublicRepoOnce() string { + cloneOnce.Do(func() { + tempDir, err := os.MkdirTemp("", "ast-cli-git-test-*") + if err != nil { + cloneErr = err + return + } + + // Clone public repo: ast-vscode-extension + repoURL := "https://github.com/Checkmarx/ast-vscode-extension.git" + cmd := exec.Command("git", "clone", "--depth", "1", repoURL, tempDir) + if err := cmd.Run(); err != nil { + cloneErr = err + return + } + + // Verify .git folder exists in cloned repo + gitPath := filepath.Join(tempDir, ".git") + if _, err := os.Stat(gitPath); err != nil { + cloneErr = err + return + } + + clonedRepoPath = tempDir + }) + return clonedRepoPath +} + +func TestExcludeGitFolder_WithFlag(t *testing.T) { + repoPath := clonePublicRepoOnce() + assert.NilError(t, cloneErr, "Failed to clone public repository") + + args := []string{ + "scan", "create", + flag(params.ProjectName), getProjectNameForScanTests(), + flag(params.SourcesFlag), repoPath, + flag(params.ScanTypes), params.IacType, + flag(params.BranchFlag), "main", + flag(params.ExcludeGitFolderFlag), + flag(params.DebugFlag), + } + + var buf bytes.Buffer + log.SetOutput(&buf) + defer func() { + log.SetOutput(os.Stderr) + }() + err, _ := executeCommand(t, args...) + assert.NilError(t, err, "scan create with --exclude-git-folder should succeed") + + logText := buf.String() + assert.Assert(t, strings.Contains(logText, "The folder .git is being excluded"), "Expected .git exclusion message not found in logs") + assert.Assert(t, strings.Contains(logText, "--exclude-git-folder flag passed"), "Expected --exclude-git-folder flag confirmation message not found in logs") +} + +func TestExcludeGitFolder_IncludeCsvJson(t *testing.T) { + repoPath := clonePublicRepoOnce() + assert.NilError(t, cloneErr, "Failed to clone public repository") + + args := []string{ + "scan", "create", + flag(params.ProjectName), getProjectNameForScanTests(), + flag(params.SourcesFlag), repoPath, + flag(params.ScanTypes), params.IacType, + flag(params.BranchFlag), "main", + flag(params.ExcludeGitFolderFlag), + flag(params.DebugFlag), + } + + var buf bytes.Buffer + log.SetOutput(&buf) + defer func() { + log.SetOutput(os.Stderr) + }() + err, _ := executeCommand(t, args...) + assert.NilError(t, err, "scan create with --exclude-git-folder should succeed") + + logText := buf.String() + assert.Assert(t, strings.Contains(logText, "The folder .git is being excluded"), "Expected .git exclusion message not found in logs") + assert.Assert(t, strings.Contains(logText, "--exclude-git-folder flag passed"), "Expected --exclude-git-folder flag confirmation message not found in logs") + assert.Assert(t, strings.Contains(logText, "Included: .checkmarx/metadata.json"), "Included contributor.csv and metadata.json") +}