diff --git a/acceptance/cli/cli.go b/acceptance/cli/cli.go index 695777c5e..d47c0f9d9 100644 --- a/acceptance/cli/cli.go +++ b/acceptance/cli/cli.go @@ -859,6 +859,12 @@ func AddStepsTo(sc *godog.ScenarioContext) { sc.Step(`^the "([^"]*)" file should match the snapshot$`, matchFileSnapshot) sc.Step(`^a file named "([^"]*)" containing$`, createGenericFile) sc.Step(`^a track bundle file named "([^"]*)" containing$`, createTrackBundleFile) + sc.Before(func(ctx context.Context, _ *godog.Scenario) (context.Context, error) { + if v := os.Getenv("EC_USE_OPA"); v != "" { + ctx, _ = theEnvironmentVarilableIsSet(ctx, "EC_USE_OPA="+v) + } + return ctx, nil + }) sc.After(func(ctx context.Context, sc *godog.Scenario, err error) (context.Context, error) { if err != nil { logExecution(ctx) diff --git a/acceptance/go.mod b/acceptance/go.mod index c43120bff..7175e8502 100644 --- a/acceptance/go.mod +++ b/acceptance/go.mod @@ -25,6 +25,7 @@ require ( github.com/sigstore/cosign/v3 v3.0.4 github.com/sigstore/rekor v1.5.0 github.com/sigstore/sigstore v1.10.4 + github.com/sigstore/sigstore-go v1.1.4 github.com/stretchr/testify v1.11.1 github.com/tektoncd/cli v0.44.1 github.com/tektoncd/pipeline v1.9.2 @@ -205,7 +206,6 @@ require ( github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/sigstore/protobuf-specs v0.5.0 // indirect github.com/sigstore/rekor-tiles/v2 v2.0.1 // indirect - github.com/sigstore/sigstore-go v1.1.4 // indirect github.com/sigstore/timestamp-authority/v2 v2.0.4 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/skeema/knownhosts v1.3.1 // indirect diff --git a/cmd/validate/image.go b/cmd/validate/image.go index 336f1080d..ac3d12009 100644 --- a/cmd/validate/image.go +++ b/cmd/validate/image.go @@ -320,7 +320,8 @@ func validateImageCmd(validate imageValidationFunc) *cobra.Command { var c evaluator.Evaluator var err error if utils.IsOpaEnabled() { - c, err = newOPAEvaluator() + c, err = newOPAEvaluator( + cmd.Context(), policySources, data.policy, sourceGroup, nil) } else { // Use the unified filtering approach with the specified filter type c, err = evaluator.NewConftestEvaluatorWithFilterType( diff --git a/features/__snapshots__/validate_image_opa.snap b/features/__snapshots__/validate_image_opa.snap new file mode 100755 index 000000000..5a1761fe7 --- /dev/null +++ b/features/__snapshots__/validate_image_opa.snap @@ -0,0 +1,553 @@ + +[OPA happy day:stdout - 1] +{ + "success": true, + "components": [ + { + "name": "Unnamed", + "containerImage": "${REGISTRY}/acceptance/opa-happy-day@sha256:${REGISTRY_acceptance/opa-happy-day:latest_DIGEST}", + "source": {}, + "successes": [ + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.signature_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.syntax_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.image.signature_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "main.acceptor" + } + } + ], + "success": true, + "signatures": [ + { + "keyid": "", + "sig": "${IMAGE_SIGNATURE_acceptance/opa-happy-day}" + } + ], + "attestations": [ + { + "type": "https://in-toto.io/Statement/v0.1", + "predicateType": "https://slsa.dev/provenance/v0.2", + "predicateBuildType": "https://tekton.dev/attestations/chains/pipelinerun@v2", + "signatures": [ + { + "keyid": "", + "sig": "${ATTESTATION_SIGNATURE_acceptance/opa-happy-day}" + } + ] + } + ] + } + ], + "key": "${known_PUBLIC_KEY_JSON}", + "policy": { + "sources": [ + { + "policy": [ + "git::${GITHOST}/git/happy-day-policy.git?ref=${LATEST_COMMIT}" + ] + } + ], + "rekorUrl": "${REKOR}", + "publicKey": "${known_PUBLIC_KEY}" + }, + "ec-version": "${EC_VERSION}", + "effective-time": "${TIMESTAMP}" +} +--- + +[OPA happy day:stderr - 1] + +--- + +[OPA volatile config warnings:stdout - 1] +{ + "success": true, + "components": [ + { + "name": "Unnamed", + "containerImage": "${REGISTRY}/acceptance/opa-volatile-config@sha256:${REGISTRY_acceptance/opa-volatile-config:latest_DIGEST}", + "source": {}, + "warnings": [ + { + "msg": "Volatile exclude rule 'test.component_scoped_rule' has no expiration date set" + }, + { + "msg": "Volatile exclude rule 'test.component_scoped_rule' is scoped to component 'Unnamed'" + }, + { + "msg": "Volatile exclude rule 'test.rule_expiring_soon' will expire (effective until: ${TIMESTAMP})" + }, + { + "msg": "Volatile exclude rule 'test.rule_pending_activation' is pending activation (effective on: ${TIMESTAMP})" + }, + { + "msg": "Volatile exclude rule 'test.rule_with_no_expiration' has no expiration date set" + } + ], + "success": true, + "signatures": [ + { + "keyid": "", + "sig": "${IMAGE_SIGNATURE_acceptance/opa-volatile-config}" + } + ], + "attestations": [ + { + "type": "https://in-toto.io/Statement/v0.1", + "predicateType": "https://slsa.dev/provenance/v0.2", + "predicateBuildType": "https://tekton.dev/attestations/chains/pipelinerun@v2", + "signatures": [ + { + "keyid": "", + "sig": "${ATTESTATION_SIGNATURE_acceptance/opa-volatile-config}" + } + ] + } + ] + } + ], + "key": "${known_PUBLIC_KEY_JSON}", + "policy": { + "sources": [ + { + "name": "volatile-test-source", + "policy": [ + "git::${GITHOST}/git/volatile-config-policy.git?ref=${LATEST_COMMIT}" + ], + "volatileConfig": { + "exclude": [ + { + "value": "test.rule_with_no_expiration" + }, + { + "value": "test.rule_expiring_soon", + "effectiveUntil": "${TIMESTAMP}" + }, + { + "value": "test.rule_pending_activation", + "effectiveOn": "${TIMESTAMP}" + }, + { + "value": "test.component_scoped_rule", + "componentNames": [ + "Unnamed" + ] + } + ] + } + } + ], + "publicKey": "${known_PUBLIC_KEY}" + }, + "ec-version": "${EC_VERSION}", + "effective-time": "${TIMESTAMP}" +} +--- + +[OPA volatile config warnings:stderr - 1] + +--- + +[OPA future failure is converted to a warning:stdout - 1] +{ + "success": true, + "components": [ + { + "name": "Unnamed", + "containerImage": "${REGISTRY}/acceptance/opa-future-deny@sha256:${REGISTRY_acceptance/opa-future-deny:latest_DIGEST}", + "source": {}, + "warnings": [ + { + "msg": "Fails in 2099", + "metadata": { + "effective_on": "${TIMESTAMP}" + } + } + ], + "successes": [ + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.signature_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.syntax_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.image.signature_check" + } + } + ], + "success": true, + "signatures": [ + { + "keyid": "", + "sig": "${IMAGE_SIGNATURE_acceptance/opa-future-deny}" + } + ], + "attestations": [ + { + "type": "https://in-toto.io/Statement/v0.1", + "predicateType": "https://slsa.dev/provenance/v0.2", + "predicateBuildType": "https://tekton.dev/attestations/chains/pipelinerun@v2", + "signatures": [ + { + "keyid": "", + "sig": "${ATTESTATION_SIGNATURE_acceptance/opa-future-deny}" + } + ] + } + ] + } + ], + "key": "${known_PUBLIC_KEY_JSON}", + "policy": { + "sources": [ + { + "policy": [ + "git::${GITHOST}/git/future-deny-policy.git?ref=${LATEST_COMMIT}" + ] + } + ], + "rekorUrl": "${REKOR}", + "publicKey": "${known_PUBLIC_KEY}" + }, + "ec-version": "${EC_VERSION}", + "effective-time": "${TIMESTAMP}" +} +--- + +[OPA future failure is converted to a warning:stderr - 1] + +--- + +[OPA policy rule filtering:stdout - 1] +{ + "success": true, + "components": [ + { + "name": "Unnamed", + "containerImage": "${REGISTRY}/acceptance/opa-filtering@sha256:${REGISTRY_acceptance/opa-filtering:latest_DIGEST}", + "source": {}, + "successes": [ + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.signature_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.syntax_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.image.signature_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "filtering.always_pass" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "filtering.always_pass_with_collection" + } + } + ], + "success": true, + "signatures": [ + { + "keyid": "", + "sig": "${IMAGE_SIGNATURE_acceptance/opa-filtering}" + } + ], + "attestations": [ + { + "type": "https://in-toto.io/Statement/v0.1", + "predicateType": "https://slsa.dev/provenance/v0.2", + "predicateBuildType": "https://tekton.dev/attestations/chains/pipelinerun@v2", + "signatures": [ + { + "keyid": "", + "sig": "${ATTESTATION_SIGNATURE_acceptance/opa-filtering}" + } + ] + } + ] + } + ], + "key": "${known_PUBLIC_KEY_JSON}", + "policy": { + "sources": [ + { + "policy": [ + "git::${GITHOST}/git/happy-day-policy.git?ref=${LATEST_COMMIT}" + ], + "config": { + "exclude": [ + "filtering.always_fail", + "filtering.always_fail_with_collection" + ], + "include": [ + "@stamps", + "filtering.always_pass" + ] + } + } + ], + "rekorUrl": "${REKOR}", + "publicKey": "${known_PUBLIC_KEY}" + }, + "ec-version": "${EC_VERSION}", + "effective-time": "${TIMESTAMP}" +} +--- + +[OPA policy rule filtering:stderr - 1] + +--- + +[OPA rejection:stdout - 1] +{ + "success": false, + "components": [ + { + "name": "Unnamed", + "containerImage": "${REGISTRY}/acceptance/opa-reject@sha256:${REGISTRY_acceptance/opa-reject:latest_DIGEST}", + "source": {}, + "violations": [ + { + "msg": "Fails always (term1)", + "metadata": { + "code": "main.reject_with_term", + "term": "term1" + } + }, + { + "msg": "Fails always (term2)", + "metadata": { + "code": "main.reject_with_term", + "term": [ + "term2", + "term3" + ] + } + }, + { + "msg": "Fails always", + "metadata": { + "code": "main.rejector" + } + } + ], + "successes": [ + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.signature_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.syntax_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.image.signature_check" + } + } + ], + "success": false, + "signatures": [ + { + "keyid": "", + "sig": "${IMAGE_SIGNATURE_acceptance/opa-reject}" + } + ], + "attestations": [ + { + "type": "https://in-toto.io/Statement/v0.1", + "predicateType": "https://slsa.dev/provenance/v0.2", + "predicateBuildType": "https://tekton.dev/attestations/chains/pipelinerun@v2", + "signatures": [ + { + "keyid": "", + "sig": "${ATTESTATION_SIGNATURE_acceptance/opa-reject}" + } + ] + } + ] + } + ], + "key": "${known_PUBLIC_KEY_JSON}", + "policy": { + "sources": [ + { + "policy": [ + "git::${GITHOST}/git/reject-policy.git?ref=${LATEST_COMMIT}" + ] + } + ], + "rekorUrl": "${REKOR}", + "publicKey": "${known_PUBLIC_KEY}" + }, + "ec-version": "${EC_VERSION}", + "effective-time": "${TIMESTAMP}" +} +--- + +[OPA rejection:stderr - 1] +Error: success criteria not met + +--- + +[OPA multiple policy sources:stdout - 1] +{ + "success": false, + "components": [ + { + "name": "Unnamed", + "containerImage": "${REGISTRY}/acceptance/opa-multiple-sources@sha256:${REGISTRY_acceptance/opa-multiple-sources:latest_DIGEST}", + "source": {}, + "violations": [ + { + "msg": "Fails always (term1)", + "metadata": { + "code": "main.reject_with_term", + "term": "term1" + } + }, + { + "msg": "Fails always (term2)", + "metadata": { + "code": "main.reject_with_term", + "term": [ + "term2", + "term3" + ] + } + }, + { + "msg": "Fails always", + "metadata": { + "code": "main.rejector" + } + } + ], + "warnings": [ + { + "msg": "Has a warning" + } + ], + "successes": [ + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.signature_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.attestation.syntax_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "builtin.image.signature_check" + } + }, + { + "msg": "Pass", + "metadata": { + "code": "main.acceptor" + } + } + ], + "success": false, + "signatures": [ + { + "keyid": "", + "sig": "${IMAGE_SIGNATURE_acceptance/opa-multiple-sources}" + } + ], + "attestations": [ + { + "type": "https://in-toto.io/Statement/v0.1", + "predicateType": "https://slsa.dev/provenance/v0.2", + "predicateBuildType": "https://tekton.dev/attestations/chains/pipelinerun@v2", + "signatures": [ + { + "keyid": "", + "sig": "${ATTESTATION_SIGNATURE_acceptance/opa-multiple-sources}" + } + ] + } + ] + } + ], + "key": "${known_PUBLIC_KEY_JSON}", + "policy": { + "sources": [ + { + "policy": [ + "git::${GITHOST}/git/repository1.git?ref=adeaf76384dd4391e18e8ce5fadef1a5c7414f06" + ] + }, + { + "policy": [ + "git::${GITHOST}/git/repository2.git?ref=7e2406bbafba94a4ecf1b5e59f9211c6597f58f7" + ] + }, + { + "policy": [ + "git::${GITHOST}/git/repository3.git?ref=${LATEST_COMMIT}" + ] + } + ], + "rekorUrl": "${REKOR}", + "publicKey": "${known_PUBLIC_KEY}" + }, + "ec-version": "${EC_VERSION}", + "effective-time": "${TIMESTAMP}" +} +--- + +[OPA multiple policy sources:stderr - 1] +Error: success criteria not met + +--- diff --git a/features/__snapshots__/validate_input_opa.snap b/features/__snapshots__/validate_input_opa.snap new file mode 100755 index 000000000..bd1e421a2 --- /dev/null +++ b/features/__snapshots__/validate_input_opa.snap @@ -0,0 +1,81 @@ + +[OPA valid policy URL:stdout - 1] +{ + "success": true, + "filepaths": [ + { + "filepath": "pipeline_definition.yaml", + "violations": [], + "warnings": [], + "successes": null, + "success": true, + "success-count": 1 + } + ], + "policy": { + "sources": [ + { + "policy": [ + "git::https://${GITHOST}/git/happy-day-policy.git" + ] + } + ] + }, + "ec-version": "${EC_VERSION}", + "effective-time": "${TIMESTAMP}" +} +--- + +[OPA valid policy URL:stderr - 1] + +--- + +[OPA policy with multiple sources:stdout - 1] +{ + "success": false, + "filepaths": [ + { + "filepath": "input.yaml", + "violations": [ + { + "msg": "ham is not delicious", + "metadata": { + "code": "ham.delicious" + } + }, + { + "msg": "spam is not true", + "metadata": { + "code": "spam.valid" + } + } + ], + "warnings": [], + "successes": null, + "success": false, + "success-count": 0 + } + ], + "policy": { + "sources": [ + { + "policy": [ + "git::https://${GITHOST}/git/ham-policy" + ] + }, + { + "policy": [ + "git::https://${GITHOST}/git/spam-policy" + ] + } + ] + }, + "ec-version": "${EC_VERSION}", + "effective-time": "${TIMESTAMP}" +} +--- + +[OPA policy with multiple sources:stderr - 1] +Error: success criteria not met + +--- diff --git a/features/validate_image_opa.feature b/features/validate_image_opa.feature new file mode 100644 index 000000000..05fb87602 --- /dev/null +++ b/features/validate_image_opa.feature @@ -0,0 +1,162 @@ +Feature: evaluate enterprise contract with OPA evaluator + The ec command line should produce correct results using the OPA evaluator + + Background: + Given the environment variable is set "EC_USE_OPA=1" + Given a stub cluster running + Given stub rekord running + Given stub registry running + Given stub git daemon running + Given stub tuf running + + Scenario: OPA happy day + Given a key pair named "known" + Given an image named "acceptance/opa-happy-day" + Given a valid image signature of "acceptance/opa-happy-day" image signed by the "known" key + Given a valid attestation of "acceptance/opa-happy-day" signed by the "known" key + Given a git repository named "happy-day-policy" with + | main.rego | examples/happy_day.rego | + Given policy configuration named "ec-policy" with specification + """ + { + "sources": [ + { + "policy": [ + "git::https://${GITHOST}/git/happy-day-policy.git" + ] + } + ] + } + """ + When ec command is run with "validate image --image ${REGISTRY}/acceptance/opa-happy-day --policy acceptance/ec-policy --public-key ${known_PUBLIC_KEY} --rekor-url ${REKOR} --show-successes --output json" + Then the exit status should be 0 + Then the output should match the snapshot + + Scenario: OPA rejection + Given a key pair named "known" + Given an image named "acceptance/opa-reject" + Given a valid image signature of "acceptance/opa-reject" image signed by the "known" key + Given a valid attestation of "acceptance/opa-reject" signed by the "known" key + Given a git repository named "reject-policy" with + | main.rego | examples/reject.rego | + Given policy configuration named "ec-policy" with specification + """ + { + "sources": [ + { + "policy": [ + "git::https://${GITHOST}/git/reject-policy.git" + ] + } + ] + } + """ + When ec command is run with "validate image --image ${REGISTRY}/acceptance/opa-reject --policy acceptance/ec-policy --public-key ${known_PUBLIC_KEY} --rekor-url ${REKOR} --show-successes --output json" + Then the exit status should be 1 + Then the output should match the snapshot + + Scenario: OPA multiple policy sources + Given a key pair named "known" + Given an image named "acceptance/opa-multiple-sources" + Given a valid image signature of "acceptance/opa-multiple-sources" image signed by the "known" key + Given a valid attestation of "acceptance/opa-multiple-sources" signed by the "known" key + Given a git repository named "repository1" with + | main.rego | examples/happy_day.rego | + Given a git repository named "repository2" with + | main.rego | examples/reject.rego | + Given a git repository named "repository3" with + | main.rego | examples/warn.rego | + Given policy configuration named "ec-policy" with specification + """ + { + "sources": [ + { "policy": ["git::https://${GITHOST}/git/repository1.git"] }, + { "policy": ["git::https://${GITHOST}/git/repository2.git"] }, + { "policy": ["git::https://${GITHOST}/git/repository3.git"] } + ] + } + """ + When ec command is run with "validate image --image ${REGISTRY}/acceptance/opa-multiple-sources --policy acceptance/ec-policy --public-key ${known_PUBLIC_KEY} --rekor-url ${REKOR} --show-successes --output json" + Then the exit status should be 1 + Then the output should match the snapshot + + Scenario: OPA policy rule filtering + Given a key pair named "known" + Given an image named "acceptance/opa-filtering" + Given a valid image signature of "acceptance/opa-filtering" image signed by the "known" key + Given a valid attestation of "acceptance/opa-filtering" signed by the "known" key + Given a git repository named "happy-day-policy" with + | filtering.rego | examples/filtering.rego | + Given policy configuration named "ec-policy" with specification + """ + { + "sources": [ + { + "policy": [ + "git::https://${GITHOST}/git/happy-day-policy.git" + ], + "config": { + "include": ["@stamps", "filtering.always_pass"], + "exclude": ["filtering.always_fail", "filtering.always_fail_with_collection"] + } + } + ] + } + """ + When ec command is run with "validate image --image ${REGISTRY}/acceptance/opa-filtering --policy acceptance/ec-policy --public-key ${known_PUBLIC_KEY} --rekor-url ${REKOR} --show-successes --output json" + Then the exit status should be 0 + Then the output should match the snapshot + + Scenario: OPA future failure is converted to a warning + Given a key pair named "known" + Given an image named "acceptance/opa-future-deny" + Given a valid image signature of "acceptance/opa-future-deny" image signed by the "known" key + Given a valid attestation of "acceptance/opa-future-deny" signed by the "known" key + Given a git repository named "future-deny-policy" with + | main.rego | examples/future_deny.rego | + When ec command is run with "validate image --image ${REGISTRY}/acceptance/opa-future-deny --policy {"sources":[{"policy":["git::https://${GITHOST}/git/future-deny-policy.git"]}]} --public-key ${known_PUBLIC_KEY} --rekor-url ${REKOR} --show-successes --output json" + Then the exit status should be 0 + Then the output should match the snapshot + + Scenario: OPA volatile config warnings + Given a key pair named "known" + Given an image named "acceptance/opa-volatile-config" + Given a valid image signature of "acceptance/opa-volatile-config" image signed by the "known" key + Given a valid attestation of "acceptance/opa-volatile-config" signed by the "known" key + Given a git repository named "volatile-config-policy" with + | main.rego | examples/volatile_config_warnings.rego | + Given policy configuration named "ec-policy" with specification + """ + { + "sources": [ + { + "name": "volatile-test-source", + "policy": [ + "git::https://${GITHOST}/git/volatile-config-policy.git" + ], + "volatileConfig": { + "exclude": [ + { + "value": "test.rule_with_no_expiration" + }, + { + "value": "test.rule_expiring_soon", + "effectiveUntil": "2099-12-31T23:59:59Z" + }, + { + "value": "test.rule_pending_activation", + "effectiveOn": "2099-01-01T00:00:00Z" + }, + { + "value": "test.component_scoped_rule", + "componentNames": ["Unnamed"] + } + ] + } + } + ] + } + """ + When ec command is run with "validate image --image ${REGISTRY}/acceptance/opa-volatile-config --policy acceptance/ec-policy --public-key ${known_PUBLIC_KEY} --ignore-rekor --output json" + Then the exit status should be 0 + Then the output should match the snapshot diff --git a/features/validate_input_opa.feature b/features/validate_input_opa.feature new file mode 100644 index 000000000..e1061962a --- /dev/null +++ b/features/validate_input_opa.feature @@ -0,0 +1,46 @@ +Feature: validate input with OPA evaluator + The ec command line should produce correct results for input validation using the OPA evaluator + + Background: + Given the environment variable is set "EC_USE_OPA=1" + Given stub git daemon running + + Scenario: OPA valid policy URL + Given a git repository named "happy-day-config" with + | policy.yaml | examples/happy_config.yaml | + Given a git repository named "happy-day-policy" with + | main.rego | examples/happy_day.rego | + Given a pipeline definition file named "pipeline_definition.yaml" containing + """ + --- + apiVersion: tekton.dev/v1 + kind: Pipeline + metadata: + name: basic-build + spec: + tasks: + - name: appstudio-init + taskRef: + name: init + version: "0.1" + """ + When ec command is run with "validate input --file pipeline_definition.yaml --policy git::https://${GITHOST}/git/happy-day-config.git --output json" + Then the exit status should be 0 + Then the output should match the snapshot + + Scenario: OPA policy with multiple sources + Given a git repository named "multiple-sources-config" with + | policy.yaml | examples/multiple_sources_config.yaml | + Given a git repository named "spam-policy" with + | main.rego | examples/spam.rego | + Given a git repository named "ham-policy" with + | main.rego | examples/ham.rego | + Given a pipeline definition file named "input.yaml" containing + """ + --- + spam: false + ham: rotten + """ + When ec command is run with "validate input --file input.yaml --policy git::https://${GITHOST}/git/multiple-sources-config.git --output json" + Then the exit status should be 1 + Then the output should match the snapshot diff --git a/internal/evaluation_target/application_snapshot_image/application_snapshot_image.go b/internal/evaluation_target/application_snapshot_image/application_snapshot_image.go index 3149d6839..73f974db1 100644 --- a/internal/evaluation_target/application_snapshot_image/application_snapshot_image.go +++ b/internal/evaluation_target/application_snapshot_image/application_snapshot_image.go @@ -419,6 +419,54 @@ type Input struct { PolicySpec ecc.EnterpriseContractPolicySpec `json:"policy_spec,omitempty"` } +// BuildInput constructs the OPA input as a Go map and JSON bytes without disk I/O. +// The JSON marshal/unmarshal round-trip ensures correct types for OPA (e.g. numbers +// as float64, consistent key ordering). +func (a *ApplicationSnapshotImage) BuildInput(_ context.Context) (map[string]any, []byte, error) { + log.Debugf("Building input for %d attestations", len(a.attestations)) + + var attestations []attestationData + for _, att := range a.attestations { + attestations = append(attestations, attestationData{ + Statement: att.Statement(), + Signatures: att.Signatures(), + }) + } + + input := Input{ + Attestations: attestations, + Image: image{ + Ref: a.reference.String(), + Signatures: a.signatures, + Config: a.configJSON, + Files: a.files, + Source: a.component.Source, + }, + AppSnapshot: a.snapshot, + ComponentName: a.component.Name, + PolicySpec: a.policySpec, + } + + if a.parentRef != nil { + input.Image.Parent = image{ + Ref: a.parentRef.String(), + Config: a.parentConfigJSON, + } + } + + inputJSON, err := json.Marshal(input) + if err != nil { + return nil, nil, fmt.Errorf("input to JSON: %w", err) + } + + var parsed map[string]any + if err := json.Unmarshal(inputJSON, &parsed); err != nil { + return nil, nil, fmt.Errorf("parse input JSON: %w", err) + } + + return parsed, inputJSON, nil +} + // WriteInputFile writes the JSON from the attestations to input.json in a random temp dir func (a *ApplicationSnapshotImage) WriteInputFile(ctx context.Context) (string, []byte, error) { log.Debugf("Attempting to write %d attestations to input file", len(a.attestations)) diff --git a/internal/evaluator/base_evaluator.go b/internal/evaluator/base_evaluator.go new file mode 100644 index 000000000..5aa696924 --- /dev/null +++ b/internal/evaluator/base_evaluator.go @@ -0,0 +1,426 @@ +// Copyright The Conforma Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package evaluator + +import ( + "context" + "fmt" + "io/fs" + "net/url" + "os" + "path" + "path/filepath" + "strings" + "time" + + ecc "github.com/conforma/crds/api/v1alpha1" + "github.com/open-policy-agent/opa/v1/ast" + log "github.com/sirupsen/logrus" + "github.com/spf13/afero" + + "github.com/conforma/cli/internal/opa" + "github.com/conforma/cli/internal/opa/rule" + "github.com/conforma/cli/internal/policy/source" + "github.com/conforma/cli/internal/utils" +) + +type basePolicyEvaluator struct { + policySources []source.PolicySource + workDir string + dataDir string + policyDir string + policy ConfigProvider + include *Criteria + exclude *Criteria + fs afero.Fs + namespace []string + source ecc.Source + policyResolver PolicyResolver + + rules policyRules + nonAnnotated nonAnnotatedRules + allRules policyRules + dataSourceDirs []string +} + +func (b *basePolicyEvaluator) initPolicyResolver(src ecc.Source, p ConfigProvider) { + b.policyResolver = NewIncludeExcludePolicyResolver(src, p) + b.include = b.policyResolver.Includes() + b.exclude = b.policyResolver.Excludes() +} + +func (b *basePolicyEvaluator) initWorkDir(ctx context.Context) error { + dir, err := utils.CreateWorkDir(b.fs) + if err != nil { + log.Debug("Failed to create work dir!") + return err + } + b.workDir = dir + b.policyDir = filepath.Join(b.workDir, "policy") + b.dataDir = filepath.Join(b.workDir, "data") + + if err := b.createDataDirectory(ctx); err != nil { + return err + } + log.Debugf("Created work dir %s", dir) + + if err := b.createCapabilitiesFile(ctx); err != nil { + return err + } + return nil +} + +func (b *basePolicyEvaluator) Destroy() { + if b.workDir != "" && os.Getenv("EC_DEBUG") == "" { + _ = b.fs.RemoveAll(b.workDir) + } +} + +func (b *basePolicyEvaluator) CapabilitiesPath() string { + return path.Join(b.workDir, "capabilities.json") +} + +func (b *basePolicyEvaluator) createDataDirectory(ctx context.Context) error { + afs := utils.FS(ctx) + exists, err := afero.DirExists(afs, b.dataDir) + if err != nil { + return err + } + if !exists { + log.Debugf("Data dir '%s' does not exist, will create.", b.dataDir) + if err := afs.MkdirAll(b.dataDir, 0755); err != nil { + return err + } + } + return createConfigJSON(ctx, b.dataDir, b.policy) +} + +func (b *basePolicyEvaluator) createCapabilitiesFile(ctx context.Context) error { + afs := utils.FS(ctx) + f, err := afs.Create(b.CapabilitiesPath()) + if err != nil { + return err + } + defer f.Close() + + data, err := strictCapabilities(ctx) + if err != nil { + return err + } + + if _, err := f.WriteString(data); err != nil { + return err + } + log.Debugf("Capabilities file written to %s", f.Name()) + return nil +} + +func (b *basePolicyEvaluator) downloadAndInspectPolicies(ctx context.Context) error { + b.rules = policyRules{} + b.nonAnnotated = nonAnnotatedRules{} + b.dataSourceDirs = []string{} + + for _, s := range b.policySources { + dir, err := s.GetPolicy(ctx, b.workDir, false) + if err != nil { + log.Debugf("Unable to download source from %s!", s.PolicyUrl()) + return err + } + if s.Subdir() == "data" { + b.dataSourceDirs = append(b.dataSourceDirs, dir) + } + + annotations := []*ast.AnnotationsRef{} + afs := utils.FS(ctx) + if s.Subdir() == "policy" { + annotations, err = opa.InspectDir(afs, dir) + if err != nil { + errMsg := err + if err.Error() == "no rego files found in policy subdirectory" { + policyURL, err := url.Parse(s.PolicyUrl()) + if err != nil { + return errMsg + } + pos := strings.LastIndex(policyURL.Path, ".") + if pos == -1 { + if (policyURL.Host == "github.com" || policyURL.Host == "gitlab.com") && (policyURL.Scheme == "https" || policyURL.Scheme == "http") { + log.Debug("Git Hub or GitLab, http transport, and no file extension, this could be a problem.") + errMsg = fmt.Errorf("%s.\nYou've specified a %s URL with an %s:// scheme.\nDid you mean: %s instead?", errMsg, policyURL.Hostname(), policyURL.Scheme, fmt.Sprint(policyURL.Host+policyURL.RequestURI())) + } + } + } + return errMsg + } + } + + for _, a := range annotations { + if a.Annotations != nil { + if err := b.rules.collect(a); err != nil { + return err + } + } else { + ruleRef := a.GetRule() + if ruleRef != nil { + packageName := "" + if len(a.Path) > 1 { + if len(a.Path) >= 2 { + packageName = strings.ReplaceAll(a.Path[1].String(), `"`, "") + } + } + + code := extractCodeFromRuleBody(ruleRef) + if code == "" { + shortName := ruleRef.Head.Name.String() + code = fmt.Sprintf("%s.%s", packageName, shortName) + } + + log.Debugf("Non-annotated rule: packageName=%s, code=%s", packageName, code) + b.nonAnnotated[code] = true + } + } + } + } + + b.allRules = make(policyRules) + for code, r := range b.rules { + b.allRules[code] = r + } + for code := range b.nonAnnotated { + parts := strings.Split(code, ".") + if len(parts) >= 2 { + packageName := parts[len(parts)-2] + shortName := parts[len(parts)-1] + b.allRules[code] = rule.Info{ + Code: code, + Package: packageName, + ShortName: shortName, + } + } + } + + return nil +} + +func (b *basePolicyEvaluator) prepareDataDirs(ctx context.Context) ([]string, error) { + dirsWithDataFiles := make(map[string]bool) + + for _, dataSourceDir := range b.dataSourceDirs { + err := fs.WalkDir(opaWrapperFs{afs: b.fs}, dataSourceDir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() { + ext := filepath.Ext(d.Name()) + if ext == ".json" || ext == ".yaml" || ext == ".yml" { + dirsWithDataFiles[filepath.Dir(p)] = true + } + } + return nil + }) + if err != nil { + continue + } + } + + err := fs.WalkDir(opaWrapperFs{afs: b.fs}, b.dataDir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if p == b.dataDir { + return nil + } + if !d.IsDir() { + ext := filepath.Ext(d.Name()) + if ext == ".json" || ext == ".yaml" || ext == ".yml" { + dirsWithDataFiles[filepath.Dir(p)] = true + } + } + return nil + }) + if err != nil { + return nil, err + } + + dataDirs := make([]string, 0, len(dirsWithDataFiles)) + for dir := range dirsWithDataFiles { + dataDirs = append(dataDirs, dir) + } + return dataDirs, nil +} + +func (b *basePolicyEvaluator) resolveFilteredNamespaces(target EvaluationTarget) []string { + if b.policyResolver != nil { + policyResolution := b.policyResolver.ResolvePolicy(b.allRules, target.Target) + var ns []string + for pkg := range policyResolution.IncludedPackages { + ns = append(ns, pkg) + } + log.Debugf("Policy resolution: %d packages included", + len(policyResolution.IncludedPackages)) + return ns + } + return nil +} + +func (b *basePolicyEvaluator) postProcessResults(ctx context.Context, runResults []Outcome, target EvaluationTarget) ([]Outcome, error) { + effectiveTime := b.policy.EffectiveTime() + ctx = context.WithValue(ctx, effectiveTimeKey, effectiveTime) + + totalRules := 0 + + missingIncludes := map[string]bool{} + for _, defaultItem := range b.include.defaultItems { + missingIncludes[defaultItem] = true + } + for _, digestItems := range b.include.digestItems { + for _, digestItem := range digestItems { + missingIncludes[digestItem] = true + } + } + + var results []Outcome + for _, result := range runResults { + unifiedFilter := NewUnifiedPostEvaluationFilter(b.policyResolver) + + allResults := []Result{} + allResults = append(allResults, result.Warnings...) + allResults = append(allResults, result.Failures...) + allResults = append(allResults, result.Exceptions...) + allResults = append(allResults, result.Skipped...) + + for j := range allResults { + addRuleMetadata(ctx, &allResults[j], b.rules) + } + + filteredResults, updatedMissingIncludes := unifiedFilter.FilterResults( + allResults, b.allRules, target.Target, target.ComponentName, missingIncludes, effectiveTime) + missingIncludes = updatedMissingIncludes + + warnings, failures, exceptions, skipped := unifiedFilter.CategorizeResults( + filteredResults, result, effectiveTime) + + result.Warnings = warnings + result.Failures = failures + result.Exceptions = exceptions + result.Skipped = skipped + + result.Successes = b.computeSuccesses(result, b.rules, target.Target, target.ComponentName, missingIncludes, unifiedFilter, effectiveTime) + + totalRules += len(result.Warnings) + len(result.Failures) + len(result.Successes) + results = append(results, result) + } + + for missingInclude, isMissing := range missingIncludes { + if isMissing { + results = append(results, Outcome{ + Warnings: []Result{{ + Message: fmt.Sprintf("Include criterion '%s' doesn't match any policy rule", missingInclude), + }}, + }) + } + } + + trim(&results) + + if totalRules == 0 { + log.Error("no successes, warnings, or failures, check input") + return nil, fmt.Errorf("no successes, warnings, or failures, check input") + } + + return results, nil +} + +func (b *basePolicyEvaluator) computeSuccesses( + result Outcome, + rules policyRules, + imageRef string, + componentName string, + missingIncludes map[string]bool, + unifiedFilter PostEvaluationFilter, + effectiveTime time.Time, +) []Result { + seenRules := map[string]bool{} + for _, outcomes := range [][]Result{result.Failures, result.Warnings, result.Skipped, result.Exceptions} { + for _, r := range outcomes { + if code, ok := r.Metadata[metadataCode].(string); ok { + seenRules[code] = true + } + } + } + + var successes []Result + if l := len(rules); l > 0 { + successes = make([]Result, 0, l) + } + + for code, r := range rules { + if seenRules[code] { + continue + } + if r.Package != result.Namespace { + continue + } + + success := Result{ + Message: "Pass", + Metadata: map[string]interface{}{ + metadataCode: code, + }, + } + if r.Title != "" { + success.Metadata[metadataTitle] = r.Title + } + if r.Description != "" { + success.Metadata[metadataDescription] = r.Description + } + if len(r.Collections) > 0 { + success.Metadata[metadataCollections] = r.Collections + } + if len(r.DependsOn) > 0 { + success.Metadata[metadataDependsOn] = r.DependsOn + } + + if unifiedFilter != nil { + filteredResults, _ := unifiedFilter.FilterResults( + []Result{success}, rules, imageRef, componentName, missingIncludes, effectiveTime) + if len(filteredResults) == 0 { + log.Debugf("Skipping result success: %#v", success) + continue + } + } else { + if !b.isResultIncluded(success, imageRef, componentName, missingIncludes) { + log.Debugf("Skipping result success: %#v", success) + continue + } + } + + if r.EffectiveOn != "" { + success.Metadata[metadataEffectiveOn] = r.EffectiveOn + } + + successes = append(successes, success) + } + + return successes +} + +func (b *basePolicyEvaluator) isResultIncluded(result Result, imageRef string, componentName string, missingIncludes map[string]bool) bool { + ruleMatchers := LegacyMakeMatchers(result) + includeScore := LegacyScoreMatches(ruleMatchers, b.include.get(imageRef, componentName), missingIncludes) + excludeScore := LegacyScoreMatches(ruleMatchers, b.exclude.get(imageRef, componentName), map[string]bool{}) + return includeScore > excludeScore +} diff --git a/internal/evaluator/conftest_evaluator.go b/internal/evaluator/conftest_evaluator.go index f2e434ae6..29a57ffe3 100644 --- a/internal/evaluator/conftest_evaluator.go +++ b/internal/evaluator/conftest_evaluator.go @@ -661,7 +661,7 @@ func (c conftestEvaluator) Evaluate(ctx context.Context, target EvaluationTarget result.Skipped = skipped // Replace the placeholder successes slice with the actual successes. - result.Successes = c.computeSuccesses(result, rules, target.Target, target.ComponentName, missingIncludes, unifiedFilter) + result.Successes = c.computeSuccesses(result, rules, target.Target, target.ComponentName, missingIncludes, unifiedFilter, effectiveTime) totalRules += len(result.Warnings) + len(result.Failures) + len(result.Successes) @@ -802,6 +802,7 @@ func (c conftestEvaluator) computeSuccesses( componentName string, missingIncludes map[string]bool, unifiedFilter PostEvaluationFilter, + effectiveTime time.Time, ) []Result { // what rules, by code, have we seen in the Conftest results, use map to // take advantage of hashing for quicker lookup @@ -858,7 +859,7 @@ func (c conftestEvaluator) computeSuccesses( if unifiedFilter != nil { // Use the unified filter to check if this success should be included filteredResults, _ := unifiedFilter.FilterResults( - []Result{success}, rules, imageRef, componentName, missingIncludes, time.Now()) + []Result{success}, rules, imageRef, componentName, missingIncludes, effectiveTime) if len(filteredResults) == 0 { log.Debugf("Skipping result success: %#v", success) diff --git a/internal/evaluator/conftest_evaluator_unit_data_test.go b/internal/evaluator/conftest_evaluator_unit_data_test.go index b7abed70c..a2e8fd4cf 100644 --- a/internal/evaluator/conftest_evaluator_unit_data_test.go +++ b/internal/evaluator/conftest_evaluator_unit_data_test.go @@ -90,14 +90,11 @@ func TestPrepareDataDirs(t *testing.T) { require.NoError(t, afero.WriteFile(fs, fullPath, []byte("test"), 0644)) } - // Create evaluator instance evaluator := conftestEvaluator{ dataDir: dataDir, fs: fs, } - // Call prepareDataDirs with the base data directory as data source - // In real usage, dataSourceDirs would be the directories returned by GetPolicy actualDirs, err := evaluator.prepareDataDirs(ctx, []string{dataDir}) require.NoError(t, err) diff --git a/internal/evaluator/conftest_evaluator_unit_filtering_test.go b/internal/evaluator/conftest_evaluator_unit_filtering_test.go index b8ef21ac7..471efcb21 100644 --- a/internal/evaluator/conftest_evaluator_unit_filtering_test.go +++ b/internal/evaluator/conftest_evaluator_unit_filtering_test.go @@ -28,6 +28,7 @@ package evaluator import ( "testing" + "time" ecc "github.com/conforma/crds/api/v1alpha1" "github.com/stretchr/testify/assert" @@ -532,6 +533,7 @@ func TestComputeSuccessesLegacyFallback(t *testing.T) { tt.componentName, tt.missingIncludes, nil, // nil unifiedFilter triggers the legacy fallback path + time.Now(), ) assert.Equal(t, tt.expectedCount, len(successes), "unexpected number of successes") diff --git a/internal/evaluator/evaluator.go b/internal/evaluator/evaluator.go index 793cfe084..ee19225c9 100644 --- a/internal/evaluator/evaluator.go +++ b/internal/evaluator/evaluator.go @@ -24,6 +24,7 @@ type EvaluationTarget struct { Inputs []string Target string ComponentName string + ParsedInput map[string]any } type Evaluator interface { diff --git a/internal/evaluator/evaluator_comparison_test.go b/internal/evaluator/evaluator_comparison_test.go new file mode 100644 index 000000000..fefdd8317 --- /dev/null +++ b/internal/evaluator/evaluator_comparison_test.go @@ -0,0 +1,538 @@ +// Copyright The Conforma Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build integration + +package evaluator + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "sort" + "testing" + "time" + + ecc "github.com/conforma/crds/api/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/conforma/cli/internal/policy" + "github.com/conforma/cli/internal/policy/source" +) + +type evaluatorPair struct { + conftest Evaluator + opa Evaluator +} + +func setupEvaluatorPair(t *testing.T, policyContent string, src ecc.Source) evaluatorPair { + t.Helper() + + tmpDir := t.TempDir() + policyDir := filepath.Join(tmpDir, "policy") + require.NoError(t, os.MkdirAll(policyDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(policyDir, "policy.rego"), []byte(policyContent), 0o600)) + + policySource := &source.PolicyUrl{ + Url: "file://" + policyDir, + Kind: source.PolicyKind, + } + + configProvider := &mockConfigProvider{} + configProvider.On("EffectiveTime").Return(time.Now()) + configProvider.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) + configProvider.On("Spec").Return(ecc.EnterpriseContractPolicySpec{ + Sources: []ecc.Source{{ + Policy: []string{"file://" + policyDir}, + }}, + }) + + ctx := context.Background() + sources := []source.PolicySource{policySource} + + conftestEval, err := NewConftestEvaluator(ctx, sources, configProvider, src) + require.NoError(t, err) + t.Cleanup(conftestEval.Destroy) + + opaEval, err := NewOPAEvaluator(ctx, sources, configProvider, src, nil) + require.NoError(t, err) + t.Cleanup(opaEval.Destroy) + + return evaluatorPair{conftest: conftestEval, opa: opaEval} +} + +func writeInput(t *testing.T, data map[string]any) string { + t.Helper() + inputBytes, err := json.Marshal(data) + require.NoError(t, err) + inputPath := filepath.Join(t.TempDir(), "input.json") + require.NoError(t, os.WriteFile(inputPath, inputBytes, 0o600)) + return inputPath +} + +type outcomeSummary struct { + failureCodes []string + warningCodes []string + successCodes []string + failureMsgs []string + warningMsgs []string + exceptionMsgs []string +} + +func summarizeOutcomes(outcomes []Outcome) outcomeSummary { + var s outcomeSummary + for _, o := range outcomes { + for _, f := range o.Failures { + if code, ok := f.Metadata["code"].(string); ok { + s.failureCodes = append(s.failureCodes, code) + } + s.failureMsgs = append(s.failureMsgs, f.Message) + } + for _, w := range o.Warnings { + if code, ok := w.Metadata["code"].(string); ok { + s.warningCodes = append(s.warningCodes, code) + } + s.warningMsgs = append(s.warningMsgs, w.Message) + } + for _, sc := range o.Successes { + if code, ok := sc.Metadata["code"].(string); ok { + s.successCodes = append(s.successCodes, code) + } + } + for _, e := range o.Exceptions { + s.exceptionMsgs = append(s.exceptionMsgs, e.Message) + } + } + sort.Strings(s.failureCodes) + sort.Strings(s.warningCodes) + sort.Strings(s.successCodes) + sort.Strings(s.failureMsgs) + sort.Strings(s.warningMsgs) + sort.Strings(s.exceptionMsgs) + return s +} + +func assertSameOutcomes(t *testing.T, label string, conftestResults, opaResults []Outcome) { + t.Helper() + cs := summarizeOutcomes(conftestResults) + os := summarizeOutcomes(opaResults) + + assert.Equal(t, cs.failureCodes, os.failureCodes, "%s: failure codes differ", label) + assert.Equal(t, cs.warningCodes, os.warningCodes, "%s: warning codes differ", label) + assert.Equal(t, cs.successCodes, os.successCodes, "%s: success codes differ", label) + assert.Equal(t, cs.failureMsgs, os.failureMsgs, "%s: failure messages differ", label) + assert.Equal(t, cs.warningMsgs, os.warningMsgs, "%s: warning messages differ", label) +} + +func TestComparisonDenyRule(t *testing.T) { + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Always deny +# custom: +# short_name: always_deny +deny contains result if { + result := { + "code": "main.always_deny", + "msg": "This always fails", + } +} +` + pair := setupEvaluatorPair(t, policyContent, ecc.Source{}) + ctx := context.Background() + inputPath := writeInput(t, map[string]any{"test": "value"}) + + target := EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "image:latest", + } + + conftestResults, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + + opaResults, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "always-deny", conftestResults, opaResults) + + cs := summarizeOutcomes(conftestResults) + assert.Contains(t, cs.failureCodes, "main.always_deny") +} + +func TestComparisonConditionalDeny(t *testing.T) { + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Conditional deny +# custom: +# short_name: conditional_deny +deny contains result if { + input.should_fail == true + result := { + "code": "main.conditional_deny", + "msg": "Conditional failure triggered", + } +} +` + pair := setupEvaluatorPair(t, policyContent, ecc.Source{}) + ctx := context.Background() + + t.Run("triggered", func(t *testing.T) { + inputPath := writeInput(t, map[string]any{"should_fail": true}) + target := EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"} + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "conditional-deny-triggered", cr, or) + assert.Contains(t, summarizeOutcomes(cr).failureCodes, "main.conditional_deny") + }) + + t.Run("not triggered", func(t *testing.T) { + inputPath := writeInput(t, map[string]any{"should_fail": false}) + target := EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"} + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "conditional-deny-not-triggered", cr, or) + assert.Contains(t, summarizeOutcomes(cr).successCodes, "main.conditional_deny") + }) +} + +func TestComparisonWarnRule(t *testing.T) { + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Always warn +# custom: +# short_name: always_warn +warn contains result if { + result := { + "code": "main.always_warn", + "msg": "This is a warning", + } +} +` + pair := setupEvaluatorPair(t, policyContent, ecc.Source{}) + ctx := context.Background() + inputPath := writeInput(t, map[string]any{"test": "value"}) + + target := EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"} + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "always-warn", cr, or) + assert.Contains(t, summarizeOutcomes(cr).warningCodes, "main.always_warn") +} + +func TestComparisonMixedDenyAndWarn(t *testing.T) { + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Deny rule +# custom: +# short_name: deny_rule +deny contains result if { + input.fail == true + result := { + "code": "main.deny_rule", + "msg": "Failure detected", + } +} + +# METADATA +# title: Warn rule +# custom: +# short_name: warn_rule +warn contains result if { + input.warn == true + result := { + "code": "main.warn_rule", + "msg": "Warning detected", + } +} +` + pair := setupEvaluatorPair(t, policyContent, ecc.Source{}) + ctx := context.Background() + + t.Run("both triggered", func(t *testing.T) { + inputPath := writeInput(t, map[string]any{"fail": true, "warn": true}) + target := EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"} + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "both-triggered", cr, or) + + s := summarizeOutcomes(cr) + assert.Contains(t, s.failureCodes, "main.deny_rule") + assert.Contains(t, s.warningCodes, "main.warn_rule") + }) + + t.Run("none triggered", func(t *testing.T) { + inputPath := writeInput(t, map[string]any{"fail": false, "warn": false}) + target := EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"} + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "none-triggered", cr, or) + + s := summarizeOutcomes(cr) + assert.Empty(t, s.failureCodes) + assert.Empty(t, s.warningCodes) + assert.NotEmpty(t, s.successCodes) + }) +} + +func TestComparisonMultipleDenyRules(t *testing.T) { + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Check Alpha +# custom: +# short_name: check_alpha +deny contains result if { + input.alpha == true + result := { + "code": "main.check_alpha", + "msg": "Alpha check failed", + } +} + +# METADATA +# title: Check Beta +# custom: +# short_name: check_beta +deny contains result if { + input.beta == true + result := { + "code": "main.check_beta", + "msg": "Beta check failed", + } +} + +# METADATA +# title: Check Gamma +# custom: +# short_name: check_gamma +deny contains result if { + input.gamma == true + result := { + "code": "main.check_gamma", + "msg": "Gamma check failed", + } +} +` + pair := setupEvaluatorPair(t, policyContent, ecc.Source{}) + ctx := context.Background() + + t.Run("all fail", func(t *testing.T) { + inputPath := writeInput(t, map[string]any{"alpha": true, "beta": true, "gamma": true}) + target := EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"} + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "all-fail", cr, or) + s := summarizeOutcomes(cr) + assert.Len(t, s.failureCodes, 3) + assert.Empty(t, s.successCodes) + }) + + t.Run("partial fail", func(t *testing.T) { + inputPath := writeInput(t, map[string]any{"alpha": true, "beta": false, "gamma": true}) + target := EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"} + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "partial-fail", cr, or) + s := summarizeOutcomes(cr) + assert.Len(t, s.failureCodes, 2) + assert.Contains(t, s.successCodes, "main.check_beta") + }) + + t.Run("all pass", func(t *testing.T) { + inputPath := writeInput(t, map[string]any{"alpha": false, "beta": false, "gamma": false}) + target := EvaluationTarget{Inputs: []string{inputPath}, Target: "image:latest"} + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "all-pass", cr, or) + s := summarizeOutcomes(cr) + assert.Empty(t, s.failureCodes) + assert.Len(t, s.successCodes, 3) + }) +} + +func TestComparisonWithParsedInput(t *testing.T) { + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Image ref check +# custom: +# short_name: image_ref_check +deny contains result if { + input.image.ref == "bad:latest" + result := { + "code": "main.image_ref_check", + "msg": "Bad image reference", + } +} +` + pair := setupEvaluatorPair(t, policyContent, ecc.Source{}) + ctx := context.Background() + + inputData := map[string]any{ + "image": map[string]any{"ref": "bad:latest"}, + } + + // Conftest needs file-based input; OPA supports ParsedInput. + // Use file input for conftest, parsed input for OPA, then compare. + inputPath := writeInput(t, inputData) + + cr, err := pair.conftest.Evaluate(ctx, EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "bad:latest", + }) + require.NoError(t, err) + + or, err := pair.opa.Evaluate(ctx, EvaluationTarget{ + ParsedInput: inputData, + Target: "bad:latest", + }) + require.NoError(t, err) + + assertSameOutcomes(t, "parsed-vs-file-input", cr, or) + assert.Contains(t, summarizeOutcomes(cr).failureCodes, "main.image_ref_check") +} + +func TestComparisonWithComponentNameFiltering(t *testing.T) { + policyContent := `package test + +import rego.v1 + +# METADATA +# title: Check A +# custom: +# short_name: check_a +deny contains result if { + result := { + "code": "test.check_a", + "msg": "Check A fails", + } +} + +# METADATA +# title: Check B +# custom: +# short_name: check_b +deny contains result if { + result := { + "code": "test.check_b", + "msg": "Check B fails", + } +} +` + src := ecc.Source{ + VolatileConfig: &ecc.VolatileSourceConfig{ + Exclude: []ecc.VolatileCriteria{ + { + Value: "test.check_a", + ComponentNames: []ecc.ComponentName{"excluded-comp"}, + EffectiveOn: "2024-01-01T00:00:00Z", + EffectiveUntil: "2030-01-01T00:00:00Z", + }, + }, + }, + } + + pair := setupEvaluatorPair(t, policyContent, src) + ctx := context.Background() + inputPath := writeInput(t, map[string]any{"test": true}) + + t.Run("excluded component", func(t *testing.T) { + target := EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "quay.io/img@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + ComponentName: "excluded-comp", + } + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "excluded-component", cr, or) + + s := summarizeOutcomes(cr) + assert.NotContains(t, s.failureCodes, "test.check_a", "check_a should be excluded") + assert.Contains(t, s.failureCodes, "test.check_b", "check_b should remain") + }) + + t.Run("non-excluded component", func(t *testing.T) { + target := EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "quay.io/img@sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", + ComponentName: "other-comp", + } + + cr, err := pair.conftest.Evaluate(ctx, target) + require.NoError(t, err) + or, err := pair.opa.Evaluate(ctx, target) + require.NoError(t, err) + + assertSameOutcomes(t, "non-excluded-component", cr, or) + + s := summarizeOutcomes(cr) + assert.Contains(t, s.failureCodes, "test.check_a", "check_a should be present") + assert.Contains(t, s.failureCodes, "test.check_b", "check_b should be present") + }) +} diff --git a/internal/evaluator/opa_evaluator.go b/internal/evaluator/opa_evaluator.go index 0732806f4..a35e07bbf 100644 --- a/internal/evaluator/opa_evaluator.go +++ b/internal/evaluator/opa_evaluator.go @@ -17,33 +17,374 @@ package evaluator import ( + "bytes" "context" + "fmt" "os" - "path" + "path/filepath" + "regexp" + "runtime/trace" + "strings" + "sync" - "github.com/spf13/afero" + ecc "github.com/conforma/crds/api/v1alpha1" + conftestParser "github.com/open-policy-agent/conftest/parser" + conftest "github.com/open-policy-agent/conftest/policy" + "github.com/open-policy-agent/opa/v1/rego" + "github.com/open-policy-agent/opa/v1/topdown/print" + log "github.com/sirupsen/logrus" + + "github.com/conforma/cli/internal/policy/source" + "github.com/conforma/cli/internal/tracing" + "github.com/conforma/cli/internal/utils" ) -// not sure what the properties will be yet, so setting the minimum. type opaEvaluator struct { - workDir string - fs afero.Fs + basePolicyEvaluator + + engine *conftest.Engine + opaTrace bool + initOnce *sync.Once + initErr error +} + +func NewOPAEvaluator(ctx context.Context, policySources []source.PolicySource, p ConfigProvider, src ecc.Source, namespace []string) (Evaluator, error) { + if trace.IsEnabled() { + r := trace.StartRegion(ctx, "ec:opa-create-evaluator") + defer r.End() + } + + o := &opaEvaluator{ + basePolicyEvaluator: basePolicyEvaluator{ + policySources: policySources, + policy: p, + fs: utils.FS(ctx), + namespace: namespace, + source: src, + }, + } + + o.initPolicyResolver(src, p) + + if err := o.initWorkDir(ctx); err != nil { + return nil, err + } + + o.initOnce = &sync.Once{} + + log.Debug("OPA evaluator created") + return o, nil +} + +func (o *opaEvaluator) compileEngine(ctx context.Context) error { + dataDirs, err := o.prepareDataDirs(ctx) + if err != nil { + return err + } + + capabilities, err := conftest.LoadCapabilities(o.CapabilitiesPath()) + if err != nil { + return fmt.Errorf("load capabilities: %w", err) + } + + opts := conftest.CompilerOptions{ + RegoVersion: "v1", + Capabilities: capabilities, + } + + engine, err := conftest.LoadWithData([]string{o.policyDir}, dataDirs, opts) + if err != nil { + return fmt.Errorf("load: %w", err) + } + + engine.EnableInterQueryCache() + o.opaTrace = tracing.FromContext(ctx).Enabled(tracing.Opa) + if o.opaTrace { + engine.EnableTracing() + } + + o.engine = engine + return nil +} + +func (o *opaEvaluator) ensureInitialized(ctx context.Context) error { + o.initOnce.Do(func() { + if err := o.downloadAndInspectPolicies(ctx); err != nil { + o.initErr = err + return + } + if err := o.compileEngine(ctx); err != nil { + o.initErr = err + } + }) + return o.initErr +} + +func (o *opaEvaluator) Evaluate(ctx context.Context, target EvaluationTarget) ([]Outcome, error) { + if trace.IsEnabled() { + region := trace.StartRegion(ctx, "ec:opa-evaluate") + defer region.End() + } + + if err := o.ensureInitialized(ctx); err != nil { + return nil, err + } + if o.engine == nil { + return nil, fmt.Errorf("OPA engine not compiled; ensure policies are on the real filesystem") + } + + filteredNamespaces := o.resolveFilteredNamespaces(target) + + runResults, err := o.evaluateWithEngine(ctx, target, filteredNamespaces) + if err != nil { + return nil, err + } + + return o.postProcessResults(ctx, runResults, target) } -func NewOPAEvaluator() (Evaluator, error) { - return opaEvaluator{}, nil +func (o *opaEvaluator) evaluateWithEngine(ctx context.Context, target EvaluationTarget, filteredNamespaces []string) ([]Outcome, error) { + namespacesToUse := o.namespace + if len(filteredNamespaces) > 0 { + namespacesToUse = filteredNamespaces + } else if len(namespacesToUse) == 0 { + namespacesToUse = o.engine.Namespaces() + } + + log.Debugf("Engine namespaces to use: %v", namespacesToUse) + + var configs map[string]any + if target.ParsedInput != nil { + configs = map[string]any{"": target.ParsedInput} + } else { + var err error + configs, err = opaParseInputFiles(target.Inputs) + if err != nil { + return nil, fmt.Errorf("parse inputs: %w", err) + } + } + + var results []Outcome + for _, ns := range namespacesToUse { + for filePath, config := range configs { + if subconfigs, ok := config.([]any); ok { + outcome := Outcome{FileName: filePath, Namespace: ns} + for _, subconfig := range subconfigs { + sub, err := o.queryNamespace(ctx, filePath, subconfig, ns) + if err != nil { + return nil, err + } + outcome.Successes = append(outcome.Successes, sub.Successes...) + outcome.Failures = append(outcome.Failures, sub.Failures...) + outcome.Warnings = append(outcome.Warnings, sub.Warnings...) + outcome.Exceptions = append(outcome.Exceptions, sub.Exceptions...) + } + results = append(results, outcome) + } else { + outcome, err := o.queryNamespace(ctx, filePath, config, ns) + if err != nil { + return nil, err + } + results = append(results, outcome) + } + } + } + return results, nil +} + +func opaParseInputFiles(inputs []string) (map[string]any, error) { + var files []string + for _, input := range inputs { + info, err := os.Stat(input) + if err != nil { + return nil, err + } + if info.IsDir() { + entries, err := os.ReadDir(input) + if err != nil { + return nil, err + } + for _, entry := range entries { + if !entry.IsDir() { + files = append(files, filepath.Join(input, entry.Name())) + } + } + } else { + files = append(files, input) + } + } + return conftestParser.ParseConfigurations(files) } -func (o opaEvaluator) Evaluate(ctx context.Context, target EvaluationTarget) ([]Outcome, error) { - return []Outcome{}, nil +var ( + opaFailureRx = regexp.MustCompile("^(deny|violation)(_[a-zA-Z0-9]+)*$") + opaWarningRx = regexp.MustCompile("^warn(_[a-zA-Z0-9]+)*$") +) + +func isOPAFailure(name string) bool { return opaFailureRx.MatchString(name) } +func isOPAWarning(name string) bool { return opaWarningRx.MatchString(name) } + +func stripRulePrefix(name string) string { + if name == "violation" || name == "deny" || name == "warn" { + return "" + } + name = strings.TrimPrefix(name, "violation_") + name = strings.TrimPrefix(name, "deny_") + name = strings.TrimPrefix(name, "warn_") + return name +} + +func (o *opaEvaluator) queryNamespace(ctx context.Context, fileName string, input any, namespace string) (Outcome, error) { + outcome := Outcome{ + FileName: fileName, + Namespace: namespace, + } + + var ruleNames []string + var ruleCount int + for _, module := range o.engine.Modules() { + ns := strings.Replace(module.Package.Path.String(), "data.", "", 1) + if ns != namespace { + continue + } + for _, r := range module.Rules { + name := r.Head.Name.String() + if !isOPAFailure(name) && !isOPAWarning(name) { + continue + } + ruleCount++ + found := false + for _, existing := range ruleNames { + if strings.EqualFold(existing, name) { + found = true + break + } + } + if !found { + ruleNames = append(ruleNames, name) + } + } + } + + var successes int + for _, ruleName := range ruleNames { + exceptionQuery := fmt.Sprintf("data.%s.exception[_][_] == %q", namespace, stripRulePrefix(ruleName)) + exceptionResults, err := o.evalOPAQuery(ctx, input, exceptionQuery) + if err != nil { + return Outcome{}, fmt.Errorf("query exception: %w", err) + } + + var exceptions []Result + for _, er := range exceptionResults { + if er.Message == "" { + exceptions = append(exceptions, Result{Message: exceptionQuery}) + } + } + + ruleQuery := fmt.Sprintf("data.%s.%s", namespace, ruleName) + ruleResults, err := o.evalOPAQuery(ctx, input, ruleQuery) + if err != nil { + return Outcome{}, fmt.Errorf("query rule: %w", err) + } + + for _, rr := range ruleResults { + if len(exceptions) > 0 { + continue + } + if rr.Message == "" { + successes++ + continue + } + if isOPAFailure(ruleName) { + outcome.Failures = append(outcome.Failures, rr) + } else { + outcome.Warnings = append(outcome.Warnings, rr) + } + } + outcome.Exceptions = append(outcome.Exceptions, exceptions...) + } + + resultCount := len(outcome.Failures) + len(outcome.Warnings) + len(outcome.Exceptions) + successes + if resultCount < ruleCount { + successes += ruleCount - resultCount + } + outcome.Successes = make([]Result, successes) + + return outcome, nil } -func (o opaEvaluator) Destroy() { - if o.workDir != "" && os.Getenv("EC_DEBUG") == "" { - _ = o.fs.RemoveAll(o.workDir) +func (o *opaEvaluator) evalOPAQuery(ctx context.Context, input any, query string) ([]Result, error) { + ph := opaPrintHook{s: &[]string{}} + options := []func(r *rego.Rego){ + rego.Input(input), + rego.Query(query), + rego.Compiler(o.engine.Compiler()), + rego.Store(o.engine.Store()), + rego.Trace(o.opaTrace), + rego.PrintHook(ph), + } + + regoInstance := rego.New(options...) + resultSet, err := regoInstance.Eval(ctx) + if err != nil { + return nil, fmt.Errorf("evaluating policy: %w", err) + } + + if o.opaTrace && log.IsLevelEnabled(log.TraceLevel) { + buf := new(bytes.Buffer) + rego.PrintTrace(buf, regoInstance) + for _, line := range strings.Split(buf.String(), "\n") { + if len(line) > 0 { + log.Tracef("[%s] %s", query, line) + } + } } + if log.IsLevelEnabled(log.DebugLevel) { + for _, out := range *ph.s { + log.Debugf("[%s] %s", query, out) + } + } + + var results []Result + for _, result := range resultSet { + for _, expression := range result.Expressions { + expressionValues, ok := expression.Value.([]any) + if !ok || len(expressionValues) == 0 { + results = append(results, Result{}) + continue + } + for _, v := range expressionValues { + switch val := v.(type) { + case string: + results = append(results, Result{ + Message: val, + Metadata: map[string]any{}, + }) + case map[string]any: + msg, _ := val["msg"].(string) + metadata := make(map[string]any) + for k, v := range val { + if k != "msg" { + metadata[k] = v + } + } + results = append(results, Result{ + Message: msg, + Metadata: metadata, + }) + } + } + } + } + + return results, nil +} + +type opaPrintHook struct { + s *[]string } -func (o opaEvaluator) CapabilitiesPath() string { - return path.Join(o.workDir, "capabilities.json") +func (ph opaPrintHook) Print(pctx print.Context, msg string) error { + *ph.s = append(*ph.s, fmt.Sprintf("%v: %s", pctx.Location, msg)) + return nil } diff --git a/internal/evaluator/opa_evaluator_integration_test.go b/internal/evaluator/opa_evaluator_integration_test.go new file mode 100644 index 000000000..b9d575f0e --- /dev/null +++ b/internal/evaluator/opa_evaluator_integration_test.go @@ -0,0 +1,476 @@ +// Copyright The Conforma Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build integration + +package evaluator + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + ecc "github.com/conforma/crds/api/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/conforma/cli/internal/policy" + "github.com/conforma/cli/internal/policy/source" +) + +func TestOPAEvaluatorIntegrationBasic(t *testing.T) { + ctx := context.Background() + + tmpDir := t.TempDir() + policyDir := filepath.Join(tmpDir, "policy") + require.NoError(t, os.MkdirAll(policyDir, 0o755)) + + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Always deny +# custom: +# short_name: always_deny +deny contains result if { + result := { + "code": "main.always_deny", + "msg": "This always fails", + } +} +` + require.NoError(t, os.WriteFile(filepath.Join(policyDir, "policy.rego"), []byte(policyContent), 0o600)) + + policySource := &source.PolicyUrl{ + Url: "file://" + policyDir, + Kind: source.PolicyKind, + } + + configProvider := &mockConfigProvider{} + configProvider.On("EffectiveTime").Return(time.Now()) + configProvider.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) + configProvider.On("Spec").Return(ecc.EnterpriseContractPolicySpec{ + Sources: []ecc.Source{{ + Policy: []string{"file://" + policyDir}, + }}, + }) + + evaluator, err := NewOPAEvaluator(ctx, []source.PolicySource{policySource}, configProvider, ecc.Source{}, nil) + require.NoError(t, err) + defer evaluator.Destroy() + + assert.NotNil(t, evaluator) + assert.NotEmpty(t, evaluator.CapabilitiesPath()) +} + +func TestOPAEvaluatorIntegrationWithTestData(t *testing.T) { + ctx := context.Background() + + tmpDir := t.TempDir() + policyDir := filepath.Join(tmpDir, "policy") + require.NoError(t, os.MkdirAll(policyDir, 0o755)) + + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Test deny +# custom: +# short_name: test_deny +deny contains result if { + result := { + "code": "main.test_deny", + "msg": "Test value found", + } +} +` + require.NoError(t, os.WriteFile(filepath.Join(policyDir, "policy.rego"), []byte(policyContent), 0o600)) + + policySource := &source.PolicyUrl{ + Url: "file://" + policyDir, + Kind: source.PolicyKind, + } + + configProvider := &mockConfigProvider{} + configProvider.On("EffectiveTime").Return(time.Now()) + configProvider.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) + configProvider.On("Spec").Return(ecc.EnterpriseContractPolicySpec{ + Sources: []ecc.Source{{ + Policy: []string{"file://" + policyDir}, + }}, + }) + + evaluator, err := NewOPAEvaluator(ctx, []source.PolicySource{policySource}, configProvider, ecc.Source{}, nil) + require.NoError(t, err) + defer evaluator.Destroy() + + inputData := map[string]any{"test": "value"} + inputBytes, err := json.Marshal(inputData) + require.NoError(t, err) + inputPath := filepath.Join(tmpDir, "input.json") + require.NoError(t, os.WriteFile(inputPath, inputBytes, 0o600)) + + target := EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "test-image:latest", + } + + results, err := evaluator.Evaluate(ctx, target) + require.NoError(t, err) + require.NotEmpty(t, results) + + hasFailure := false + for _, outcome := range results { + for _, failure := range outcome.Failures { + if failure.Message == "Test value found" { + hasFailure = true + } + } + } + assert.True(t, hasFailure, "Expected deny rule to produce a failure") +} + +func TestOPAEvaluatorIntegrationDenyWarnException(t *testing.T) { + ctx := context.Background() + + tmpDir := t.TempDir() + policyDir := filepath.Join(tmpDir, "policy") + require.NoError(t, os.MkdirAll(policyDir, 0o755)) + + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Deny check +# custom: +# short_name: deny_check +deny contains result if { + input.should_deny == true + result := { + "code": "main.deny_check", + "msg": "Deny triggered", + } +} + +# METADATA +# title: Warn check +# custom: +# short_name: warn_check +warn contains result if { + input.should_warn == true + result := { + "code": "main.warn_check", + "msg": "Warning triggered", + } +} +` + require.NoError(t, os.WriteFile(filepath.Join(policyDir, "policy.rego"), []byte(policyContent), 0o600)) + + policySource := &source.PolicyUrl{ + Url: "file://" + policyDir, + Kind: source.PolicyKind, + } + + configProvider := &mockConfigProvider{} + configProvider.On("EffectiveTime").Return(time.Now()) + configProvider.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) + configProvider.On("Spec").Return(ecc.EnterpriseContractPolicySpec{ + Sources: []ecc.Source{{ + Policy: []string{"file://" + policyDir}, + }}, + }) + + evaluator, err := NewOPAEvaluator(ctx, []source.PolicySource{policySource}, configProvider, ecc.Source{}, nil) + require.NoError(t, err) + defer evaluator.Destroy() + + t.Run("deny and warn both triggered", func(t *testing.T) { + inputData := map[string]any{"should_deny": true, "should_warn": true} + inputBytes, err := json.Marshal(inputData) + require.NoError(t, err) + inputPath := filepath.Join(tmpDir, "input_both.json") + require.NoError(t, os.WriteFile(inputPath, inputBytes, 0o600)) + + results, err := evaluator.Evaluate(ctx, EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "image:latest", + }) + require.NoError(t, err) + + var failures, warnings int + for _, outcome := range results { + failures += len(outcome.Failures) + warnings += len(outcome.Warnings) + } + assert.Equal(t, 1, failures, "Expected 1 deny failure") + assert.Equal(t, 1, warnings, "Expected 1 warning") + }) + + t.Run("only warn triggered", func(t *testing.T) { + inputData := map[string]any{"should_deny": false, "should_warn": true} + inputBytes, err := json.Marshal(inputData) + require.NoError(t, err) + inputPath := filepath.Join(tmpDir, "input_warn.json") + require.NoError(t, os.WriteFile(inputPath, inputBytes, 0o600)) + + results, err := evaluator.Evaluate(ctx, EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "image:latest", + }) + require.NoError(t, err) + + var failures, warnings, successes int + for _, outcome := range results { + failures += len(outcome.Failures) + warnings += len(outcome.Warnings) + successes += len(outcome.Successes) + } + assert.Equal(t, 0, failures, "Expected no deny failures") + assert.Equal(t, 1, warnings, "Expected 1 warning") + assert.GreaterOrEqual(t, successes, 1, "Expected at least 1 success") + }) + + t.Run("nothing triggered produces successes", func(t *testing.T) { + inputData := map[string]any{"should_deny": false, "should_warn": false} + inputBytes, err := json.Marshal(inputData) + require.NoError(t, err) + inputPath := filepath.Join(tmpDir, "input_pass.json") + require.NoError(t, os.WriteFile(inputPath, inputBytes, 0o600)) + + results, err := evaluator.Evaluate(ctx, EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "image:latest", + }) + require.NoError(t, err) + + var failures, warnings, successes int + for _, outcome := range results { + failures += len(outcome.Failures) + warnings += len(outcome.Warnings) + successes += len(outcome.Successes) + } + assert.Equal(t, 0, failures, "Expected no failures") + assert.Equal(t, 0, warnings, "Expected no warnings") + assert.GreaterOrEqual(t, successes, 1, "Expected successes for passing rules") + }) +} + +func TestOPAEvaluatorIntegrationWithParsedInput(t *testing.T) { + ctx := context.Background() + + tmpDir := t.TempDir() + policyDir := filepath.Join(tmpDir, "policy") + require.NoError(t, os.MkdirAll(policyDir, 0o755)) + + policyContent := `package main + +import rego.v1 + +# METADATA +# title: Image check +# custom: +# short_name: image_check +deny contains result if { + input.image.ref == "bad-image:latest" + result := { + "code": "main.image_check", + "msg": "Bad image detected", + } +} +` + require.NoError(t, os.WriteFile(filepath.Join(policyDir, "policy.rego"), []byte(policyContent), 0o600)) + + policySource := &source.PolicyUrl{ + Url: "file://" + policyDir, + Kind: source.PolicyKind, + } + + configProvider := &mockConfigProvider{} + configProvider.On("EffectiveTime").Return(time.Now()) + configProvider.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) + configProvider.On("Spec").Return(ecc.EnterpriseContractPolicySpec{ + Sources: []ecc.Source{{ + Policy: []string{"file://" + policyDir}, + }}, + }) + + evaluator, err := NewOPAEvaluator(ctx, []source.PolicySource{policySource}, configProvider, ecc.Source{}, nil) + require.NoError(t, err) + defer evaluator.Destroy() + + t.Run("parsed input triggers deny", func(t *testing.T) { + results, err := evaluator.Evaluate(ctx, EvaluationTarget{ + ParsedInput: map[string]any{ + "image": map[string]any{"ref": "bad-image:latest"}, + }, + Target: "bad-image:latest", + }) + require.NoError(t, err) + + hasFailure := false + for _, outcome := range results { + for _, f := range outcome.Failures { + if f.Message == "Bad image detected" { + hasFailure = true + } + } + } + assert.True(t, hasFailure, "Expected deny rule to trigger with parsed input") + }) + + t.Run("parsed input passes", func(t *testing.T) { + results, err := evaluator.Evaluate(ctx, EvaluationTarget{ + ParsedInput: map[string]any{ + "image": map[string]any{"ref": "good-image:latest"}, + }, + Target: "good-image:latest", + }) + require.NoError(t, err) + + var failures int + for _, outcome := range results { + failures += len(outcome.Failures) + } + assert.Equal(t, 0, failures, "Expected no failures for good image") + }) +} + +func TestOPAEvaluatorIntegrationWithComponentNames(t *testing.T) { + ctx := context.Background() + + tmpDir := t.TempDir() + policyDir := filepath.Join(tmpDir, "policy") + require.NoError(t, os.MkdirAll(policyDir, 0o755)) + + policyContent := `package test + +import rego.v1 + +# METADATA +# title: Check A +# custom: +# short_name: check_a +deny contains result if { + result := { + "code": "test.check_a", + "msg": "Check A always fails" + } +} + +# METADATA +# title: Check B +# custom: +# short_name: check_b +deny contains result if { + result := { + "code": "test.check_b", + "msg": "Check B always fails" + } +} +` + require.NoError(t, os.WriteFile(filepath.Join(policyDir, "policy.rego"), []byte(policyContent), 0o600)) + + policySource := &source.PolicyUrl{ + Url: "file://" + policyDir, + Kind: source.PolicyKind, + } + + configProvider := &mockConfigProvider{} + configProvider.On("EffectiveTime").Return(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC)) + configProvider.On("SigstoreOpts").Return(policy.SigstoreOpts{}, nil) + configProvider.On("Spec").Return(ecc.EnterpriseContractPolicySpec{ + Sources: []ecc.Source{{ + Policy: []string{"file://" + policyDir}, + }}, + }) + + evaluator, err := NewOPAEvaluator(ctx, []source.PolicySource{policySource}, configProvider, ecc.Source{ + VolatileConfig: &ecc.VolatileSourceConfig{ + Exclude: []ecc.VolatileCriteria{ + { + Value: "test.check_a", + ComponentNames: []ecc.ComponentName{"comp1"}, + EffectiveOn: "2024-01-01T00:00:00Z", + EffectiveUntil: "2025-01-01T00:00:00Z", + }, + }, + }, + }, nil) + require.NoError(t, err) + defer evaluator.Destroy() + + inputData := map[string]any{"test": "value"} + inputBytes, err := json.Marshal(inputData) + require.NoError(t, err) + inputPath := filepath.Join(tmpDir, "input.json") + require.NoError(t, os.WriteFile(inputPath, inputBytes, 0o600)) + + t.Run("comp1 excludes check_a", func(t *testing.T) { + results, err := evaluator.Evaluate(ctx, EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "quay.io/repo/img@sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + ComponentName: "comp1", + }) + require.NoError(t, err) + + hasCheckA, hasCheckB := false, false + for _, outcome := range results { + for _, failure := range outcome.Failures { + if code, ok := failure.Metadata["code"].(string); ok { + if code == "test.check_a" { + hasCheckA = true + } + if code == "test.check_b" { + hasCheckB = true + } + } + } + } + assert.False(t, hasCheckA, "Expected check_a to be excluded for comp1") + assert.True(t, hasCheckB, "Expected check_b to be evaluated for comp1") + }) + + t.Run("comp2 evaluates both checks", func(t *testing.T) { + results, err := evaluator.Evaluate(ctx, EvaluationTarget{ + Inputs: []string{inputPath}, + Target: "quay.io/repo/img@sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", + ComponentName: "comp2", + }) + require.NoError(t, err) + + hasCheckA, hasCheckB := false, false + for _, outcome := range results { + for _, failure := range outcome.Failures { + if code, ok := failure.Metadata["code"].(string); ok { + if code == "test.check_a" { + hasCheckA = true + } + if code == "test.check_b" { + hasCheckB = true + } + } + } + } + assert.True(t, hasCheckA, "Expected check_a to be evaluated for comp2") + assert.True(t, hasCheckB, "Expected check_b to be evaluated for comp2") + }) +} diff --git a/internal/evaluator/opa_evaluator_test.go b/internal/evaluator/opa_evaluator_test.go index 278b42fd4..136eb837d 100644 --- a/internal/evaluator/opa_evaluator_test.go +++ b/internal/evaluator/opa_evaluator_test.go @@ -14,39 +14,35 @@ // // SPDX-License-Identifier: Apache-2.0 +//go:build unit + package evaluator import ( "context" + "encoding/json" + "fmt" "os" + "path/filepath" + "sync" "testing" + "time" + ecc "github.com/conforma/crds/api/v1alpha1" + conftest "github.com/open-policy-agent/conftest/policy" + "github.com/open-policy-agent/opa/v1/topdown/print" "github.com/spf13/afero" "github.com/stretchr/testify/assert" -) - -// TestNewOPAEvaluator tests the constructor NewOPAEvaluator. -func TestNewOPAEvaluator(t *testing.T) { - evaluator, err := NewOPAEvaluator() - assert.NoError(t, err, "Expected no error from NewOPAEvaluator") - assert.Equal(t, evaluator, opaEvaluator{}) -} + "github.com/stretchr/testify/require" -func TestEvaluate(t *testing.T) { - opaEval := opaEvaluator{} - - outcomes, err := opaEval.Evaluate(context.Background(), EvaluationTarget{}) - assert.NoError(t, err, "Expected no error from Evaluate") - assert.Equal(t, []Outcome{}, outcomes) -} + "github.com/conforma/cli/internal/opa/rule" + "github.com/conforma/cli/internal/utils" +) -// Test Destroy method of opaEvaluator. -func TestDestroy(t *testing.T) { - // Setup an in-memory filesystem +func TestOPADestroy(t *testing.T) { fs := afero.NewMemMapFs() workDir := "/tmp/workdir" - // Define test cases testCases := []struct { name string workDir string @@ -75,30 +71,29 @@ func TestDestroy(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - // Set up the environment if tc.workDir != "" { err := fs.MkdirAll(tc.workDir, 0755) - assert.NoError(t, err, "Failed to create workDir in in-memory filesystem") + assert.NoError(t, err) } if tc.EC_DEBUG { - os.Setenv("EC_DEBUG", "true") + t.Setenv("EC_DEBUG", "true") } else { + t.Setenv("EC_DEBUG", "") os.Unsetenv("EC_DEBUG") } - // Initialize the evaluator opaEval := opaEvaluator{ - workDir: tc.workDir, - fs: fs, + basePolicyEvaluator: basePolicyEvaluator{ + workDir: tc.workDir, + fs: fs, + }, } - // Call Destroy opaEval.Destroy() - // Verify the result exists, err := afero.DirExists(fs, tc.workDir) - assert.NoError(t, err, "Error checking if workDir exists after Destroy") + assert.NoError(t, err) if tc.expectRemove { assert.False(t, exists, "workDir should be removed") @@ -106,16 +101,12 @@ func TestDestroy(t *testing.T) { assert.True(t, exists, "workDir should not be removed") } - // Clean up for next test _ = fs.RemoveAll(tc.workDir) - os.Unsetenv("EC_DEBUG") }) } } -// TestCapabilitiesPath tests the CapabilitiesPath method of opaEvaluator. -func TestCapabilitiesPath(t *testing.T) { - // Define test cases +func TestOPACapabilitiesPath(t *testing.T) { testCases := []struct { name string workDir string @@ -135,20 +126,828 @@ func TestCapabilitiesPath(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - // Create a mock filesystem (though not strictly needed for this test) - fs := afero.NewMemMapFs() - - // Initialize the evaluator with test data opaEval := opaEvaluator{ - workDir: tc.workDir, - fs: fs, + basePolicyEvaluator: basePolicyEvaluator{ + workDir: tc.workDir, + fs: afero.NewMemMapFs(), + }, } - // Call CapabilitiesPath result := opaEval.CapabilitiesPath() + assert.Equal(t, tc.expected, result) + }) + } +} + +func TestIsOPAFailure(t *testing.T) { + tests := []struct { + name string + input string + expected bool + }{ + {"deny", "deny", true}, + {"deny_with_suffix", "deny_foo", true}, + {"deny_multi_suffix", "deny_foo_bar", true}, + {"violation", "violation", true}, + {"violation_with_suffix", "violation_check1", true}, + {"warn_not_failure", "warn", false}, + {"warn_suffix_not_failure", "warn_thing", false}, + {"random_name", "allow", false}, + {"deny_prefix_only", "denyall", false}, + {"empty", "", false}, + {"deny_special_chars", "deny_foo-bar", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, isOPAFailure(tt.input)) + }) + } +} + +func TestIsOPAWarning(t *testing.T) { + tests := []struct { + name string + input string + expected bool + }{ + {"warn", "warn", true}, + {"warn_with_suffix", "warn_foo", true}, + {"warn_multi_suffix", "warn_foo_bar", true}, + {"deny_not_warning", "deny", false}, + {"violation_not_warning", "violation", false}, + {"random_name", "allow", false}, + {"empty", "", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, isOPAWarning(tt.input)) + }) + } +} + +func TestStripRulePrefix(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + {"deny_bare", "deny", ""}, + {"violation_bare", "violation", ""}, + {"warn_bare", "warn", ""}, + {"deny_prefix", "deny_foo", "foo"}, + {"violation_prefix", "violation_check", "check"}, + {"warn_prefix", "warn_thing", "thing"}, + {"no_prefix", "allow", "allow"}, + {"deny_multi_part", "deny_foo_bar", "foo_bar"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, stripRulePrefix(tt.input)) + }) + } +} + +func TestOpaPrintHook(t *testing.T) { + s := &[]string{} + ph := opaPrintHook{s: s} + + err := ph.Print(print.Context{Location: nil}, "hello world") + require.NoError(t, err) + assert.Len(t, *s, 1) + assert.Contains(t, (*s)[0], "hello world") + + err = ph.Print(print.Context{Location: nil}, "second message") + require.NoError(t, err) + assert.Len(t, *s, 2) +} + +func TestOpaParseInputFiles(t *testing.T) { + t.Run("single file", func(t *testing.T) { + dir := t.TempDir() + inputFile := filepath.Join(dir, "input.json") + content := `{"image": {"ref": "registry.example.com/image:latest"}}` + require.NoError(t, os.WriteFile(inputFile, []byte(content), 0600)) + + configs, err := opaParseInputFiles([]string{inputFile}) + require.NoError(t, err) + assert.Len(t, configs, 1) + + for _, v := range configs { + m, ok := v.(map[string]any) + require.True(t, ok) + img, ok := m["image"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "registry.example.com/image:latest", img["ref"]) + } + }) + + t.Run("directory of files", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, "a.json"), + []byte(`{"key": "value_a"}`), 0600)) + require.NoError(t, os.WriteFile( + filepath.Join(dir, "b.json"), + []byte(`{"key": "value_b"}`), 0600)) + + configs, err := opaParseInputFiles([]string{dir}) + require.NoError(t, err) + assert.Len(t, configs, 2) + }) + + t.Run("nonexistent file", func(t *testing.T) { + _, err := opaParseInputFiles([]string{"/nonexistent/file.json"}) + assert.Error(t, err) + }) +} + +func setupOPAEngine(t *testing.T, policyContent string) (*conftest.Engine, string) { + t.Helper() + dir := t.TempDir() + policyDir := filepath.Join(dir, "policy") + require.NoError(t, os.MkdirAll(policyDir, 0755)) + require.NoError(t, os.WriteFile( + filepath.Join(policyDir, "policy.rego"), + []byte(policyContent), 0600)) + + capPath := filepath.Join(dir, "capabilities.json") + require.NoError(t, os.WriteFile(capPath, []byte(testCapabilities), 0600)) + + capabilities, err := conftest.LoadCapabilities(capPath) + require.NoError(t, err) + + engine, err := conftest.LoadWithData([]string{policyDir}, nil, conftest.CompilerOptions{ + RegoVersion: "v1", + Capabilities: capabilities, + }) + require.NoError(t, err) + return engine, dir +} + +func TestEvalOPAQuery(t *testing.T) { + policyContent := `package main + +import rego.v1 + +deny contains result if { + input.value == "bad" + result := "value is bad" +} + +deny_structured contains result if { + input.value == "structured" + result := { + "msg": "structured failure", + "code": "main.structured", + } +} + +warn contains result if { + input.level == "warning" + result := "this is a warning" +} +` + engine, _ := setupOPAEngine(t, policyContent) + + o := &opaEvaluator{engine: engine} + ctx := context.Background() + + t.Run("deny rule with string result", func(t *testing.T) { + results, err := o.evalOPAQuery(ctx, map[string]any{"value": "bad"}, "data.main.deny") + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "value is bad", results[0].Message) + }) + + t.Run("deny rule with structured result", func(t *testing.T) { + results, err := o.evalOPAQuery(ctx, map[string]any{"value": "structured"}, "data.main.deny_structured") + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "structured failure", results[0].Message) + assert.Equal(t, "main.structured", results[0].Metadata["code"]) + }) + + t.Run("no match returns empty result", func(t *testing.T) { + results, err := o.evalOPAQuery(ctx, map[string]any{"value": "good"}, "data.main.deny") + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "", results[0].Message) + }) + + t.Run("warn rule", func(t *testing.T) { + results, err := o.evalOPAQuery(ctx, map[string]any{"level": "warning"}, "data.main.warn") + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, "this is a warning", results[0].Message) + }) +} + +func TestQueryNamespace(t *testing.T) { + policyContent := `package test.ns + +import rego.v1 + +deny contains result if { + input.fail == true + result := { + "msg": "input failed", + "code": "test.ns.deny", + } +} + +warn contains result if { + input.warn == true + result := "warning message" +} + +deny_extra contains result if { + input.extra == true + result := "extra failure" +} +` + engine, _ := setupOPAEngine(t, policyContent) + + o := &opaEvaluator{engine: engine} + ctx := context.Background() + + t.Run("failure result", func(t *testing.T) { + outcome, err := o.queryNamespace(ctx, "test.json", map[string]any{"fail": true}, "test.ns") + require.NoError(t, err) + assert.Equal(t, "test.ns", outcome.Namespace) + assert.Equal(t, "test.json", outcome.FileName) + assert.Len(t, outcome.Failures, 1) + assert.Equal(t, "input failed", outcome.Failures[0].Message) + }) + + t.Run("warning result", func(t *testing.T) { + outcome, err := o.queryNamespace(ctx, "test.json", map[string]any{"warn": true}, "test.ns") + require.NoError(t, err) + assert.Len(t, outcome.Warnings, 1) + assert.Equal(t, "warning message", outcome.Warnings[0].Message) + }) + + t.Run("success when rules pass", func(t *testing.T) { + outcome, err := o.queryNamespace(ctx, "test.json", map[string]any{"fail": false, "warn": false, "extra": false}, "test.ns") + require.NoError(t, err) + assert.Empty(t, outcome.Failures) + assert.Empty(t, outcome.Warnings) + assert.NotEmpty(t, outcome.Successes) + }) + + t.Run("nonexistent namespace", func(t *testing.T) { + outcome, err := o.queryNamespace(ctx, "test.json", map[string]any{}, "nonexistent.ns") + require.NoError(t, err) + assert.Empty(t, outcome.Failures) + assert.Empty(t, outcome.Warnings) + assert.Empty(t, outcome.Successes) + }) + + t.Run("multiple failures", func(t *testing.T) { + outcome, err := o.queryNamespace(ctx, "test.json", map[string]any{"fail": true, "extra": true}, "test.ns") + require.NoError(t, err) + assert.Len(t, outcome.Failures, 2) + }) +} + +func TestQueryNamespaceWithExceptions(t *testing.T) { + policyContent := `package test.exc + +import rego.v1 + +deny contains result if { + input.fail == true + result := "should fail" +} + +exception contains rules if { + rules := ["", ""] +} +` + engine, _ := setupOPAEngine(t, policyContent) + + o := &opaEvaluator{engine: engine} + ctx := context.Background() + + outcome, err := o.queryNamespace(ctx, "test.json", map[string]any{"fail": true}, "test.exc") + require.NoError(t, err) + assert.Empty(t, outcome.Failures, "failures should be suppressed by exception") + assert.NotEmpty(t, outcome.Exceptions) +} + +func TestEvaluateWithEngine(t *testing.T) { + policyContent := `package eval.test + +import rego.v1 + +deny contains result if { + input.should_fail == true + result := { + "msg": "evaluation failed", + "code": "eval.test.deny", + } +} +` + engine, dir := setupOPAEngine(t, policyContent) + + t.Run("with parsed input", func(t *testing.T) { + o := &opaEvaluator{ + engine: engine, + basePolicyEvaluator: basePolicyEvaluator{ + namespace: []string{"eval.test"}, + }, + } + + target := EvaluationTarget{ + ParsedInput: map[string]any{"should_fail": true}, + } + + results, err := o.evaluateWithEngine(context.Background(), target, nil) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Len(t, results[0].Failures, 1) + assert.Equal(t, "evaluation failed", results[0].Failures[0].Message) + }) + + t.Run("with file input", func(t *testing.T) { + inputFile := filepath.Join(dir, "input.json") + require.NoError(t, os.WriteFile(inputFile, []byte(`{"should_fail": true}`), 0600)) + + o := &opaEvaluator{ + engine: engine, + basePolicyEvaluator: basePolicyEvaluator{ + namespace: []string{"eval.test"}, + }, + } + + target := EvaluationTarget{ + Inputs: []string{inputFile}, + } + + results, err := o.evaluateWithEngine(context.Background(), target, nil) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Len(t, results[0].Failures, 1) + }) - // Verify the result - assert.Equal(t, tc.expected, result, "CapabilitiesPath should return the expected path") + t.Run("with filtered namespaces", func(t *testing.T) { + o := &opaEvaluator{ + engine: engine, + basePolicyEvaluator: basePolicyEvaluator{ + namespace: []string{"some.other.ns"}, + }, + } + + target := EvaluationTarget{ + ParsedInput: map[string]any{"should_fail": true}, + } + + results, err := o.evaluateWithEngine(context.Background(), target, []string{"eval.test"}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Len(t, results[0].Failures, 1) + }) + + t.Run("uses engine namespaces when none specified", func(t *testing.T) { + o := &opaEvaluator{ + engine: engine, + basePolicyEvaluator: basePolicyEvaluator{ + namespace: nil, + }, + } + + target := EvaluationTarget{ + ParsedInput: map[string]any{"should_fail": true}, + } + + results, err := o.evaluateWithEngine(context.Background(), target, nil) + require.NoError(t, err) + assert.NotEmpty(t, results) + }) + + t.Run("with list input", func(t *testing.T) { + inputFile := filepath.Join(dir, "list_input.json") + require.NoError(t, os.WriteFile(inputFile, []byte(`[{"should_fail": true}, {"should_fail": false}]`), 0600)) + + o := &opaEvaluator{ + engine: engine, + basePolicyEvaluator: basePolicyEvaluator{ + namespace: []string{"eval.test"}, + }, + } + + target := EvaluationTarget{ + Inputs: []string{inputFile}, + } + + results, err := o.evaluateWithEngine(context.Background(), target, nil) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Len(t, results[0].Failures, 1) + }) +} + +func TestEnsureInitialized(t *testing.T) { + t.Run("returns error from init", func(t *testing.T) { + expectedErr := fmt.Errorf("init failed") + o := &opaEvaluator{ + initOnce: &sync.Once{}, + initErr: expectedErr, + } + // initOnce already "ran" since initErr is set — but sync.Once + // hasn't been used yet. Let's set it up properly. + once := &sync.Once{} + o.initOnce = once + o.initErr = nil + + // Force initialization to fail by having no policy sources + o.basePolicyEvaluator = basePolicyEvaluator{ + fs: afero.NewMemMapFs(), + } + + err := o.ensureInitialized(context.Background()) + assert.Error(t, err) + }) + + t.Run("only runs once", func(t *testing.T) { + callCount := 0 + o := &opaEvaluator{ + initOnce: &sync.Once{}, + } + o.initOnce.Do(func() { + callCount++ + o.initErr = fmt.Errorf("test error") }) + + err := o.ensureInitialized(context.Background()) + assert.Error(t, err) + assert.Equal(t, 1, callCount) + + err = o.ensureInitialized(context.Background()) + assert.Error(t, err) + assert.Equal(t, 1, callCount) + }) +} + +func TestOPAEvaluateNilEngine(t *testing.T) { + o := &opaEvaluator{ + initOnce: &sync.Once{}, + engine: nil, + } + o.initOnce.Do(func() {}) + + _, err := o.Evaluate(context.Background(), EvaluationTarget{}) + assert.Error(t, err) + assert.Contains(t, err.Error(), "OPA engine not compiled") +} + +func TestBasePolicyEvaluatorPrepareDataDirs(t *testing.T) { + fs := afero.NewMemMapFs() + ctx := utils.WithFS(context.Background(), fs) + dataDir := "/test/data" + require.NoError(t, fs.MkdirAll(dataDir, 0755)) + + require.NoError(t, fs.MkdirAll(filepath.Join(dataDir, "subdir"), 0755)) + require.NoError(t, afero.WriteFile(fs, filepath.Join(dataDir, "subdir", "data.json"), []byte("{}"), 0644)) + require.NoError(t, afero.WriteFile(fs, filepath.Join(dataDir, "config.yaml"), []byte("---"), 0644)) + require.NoError(t, afero.WriteFile(fs, filepath.Join(dataDir, "readme.txt"), []byte("skip"), 0644)) + + b := &basePolicyEvaluator{ + dataDir: dataDir, + fs: fs, + } + + dirs, err := b.prepareDataDirs(ctx) + require.NoError(t, err) + assert.Contains(t, dirs, dataDir) + assert.Contains(t, dirs, filepath.Join(dataDir, "subdir")) + assert.Len(t, dirs, 2) +} + +func TestBasePolicyEvaluatorPrepareDataDirsWithDataSourceDirs(t *testing.T) { + fs := afero.NewMemMapFs() + ctx := utils.WithFS(context.Background(), fs) + dataDir := "/test/data" + sourceDir := "/test/sources" + + require.NoError(t, fs.MkdirAll(dataDir, 0755)) + require.NoError(t, fs.MkdirAll(filepath.Join(sourceDir, "rule_data"), 0755)) + require.NoError(t, afero.WriteFile(fs, filepath.Join(sourceDir, "rule_data", "data.yml"), []byte("---"), 0644)) + + b := &basePolicyEvaluator{ + dataDir: dataDir, + fs: fs, + dataSourceDirs: []string{sourceDir}, + } + + dirs, err := b.prepareDataDirs(ctx) + require.NoError(t, err) + assert.Contains(t, dirs, filepath.Join(sourceDir, "rule_data")) +} + +func TestBasePolicyEvaluatorCreateDataDirectory(t *testing.T) { + fs := afero.NewMemMapFs() + ctx := utils.WithFS(context.Background(), fs) + ctx = withCapabilities(ctx, testCapabilities) + dataDir := "/test/data" + + config := &simpleConfigProvider{effectiveTime: time.Now()} + + b := &basePolicyEvaluator{ + dataDir: dataDir, + fs: fs, + policy: config, + } + + err := b.createDataDirectory(ctx) + require.NoError(t, err) + + exists, err := afero.DirExists(fs, dataDir) + require.NoError(t, err) + assert.True(t, exists) + + configExists, err := afero.Exists(fs, filepath.Join(dataDir, "config", "config.json")) + require.NoError(t, err) + assert.True(t, configExists) +} + +func TestBasePolicyEvaluatorCreateCapabilitiesFile(t *testing.T) { + fs := afero.NewMemMapFs() + ctx := utils.WithFS(context.Background(), fs) + ctx = withCapabilities(ctx, testCapabilities) + workDir := "/test/work" + + require.NoError(t, fs.MkdirAll(workDir, 0755)) + + b := &basePolicyEvaluator{ + workDir: workDir, + fs: fs, + } + + err := b.createCapabilitiesFile(ctx) + require.NoError(t, err) + + capPath := b.CapabilitiesPath() + exists, err := afero.Exists(fs, capPath) + require.NoError(t, err) + assert.True(t, exists) + + content, err := afero.ReadFile(fs, capPath) + require.NoError(t, err) + + var parsed map[string]any + require.NoError(t, json.Unmarshal(content, &parsed)) +} + +func TestBasePolicyEvaluatorInitWorkDir(t *testing.T) { + fs := afero.NewMemMapFs() + ctx := utils.WithFS(context.Background(), fs) + ctx = withCapabilities(ctx, testCapabilities) + + config := &simpleConfigProvider{effectiveTime: time.Now()} + + b := &basePolicyEvaluator{ + fs: fs, + policy: config, + } + + err := b.initWorkDir(ctx) + require.NoError(t, err) + assert.NotEmpty(t, b.workDir) + assert.NotEmpty(t, b.policyDir) + assert.NotEmpty(t, b.dataDir) + + exists, err := afero.DirExists(fs, b.workDir) + require.NoError(t, err) + assert.True(t, exists) +} + +func TestBasePolicyEvaluatorComputeSuccesses(t *testing.T) { + rules := policyRules{ + "test.ns.rule1": rule.Info{ + Code: "test.ns.rule1", + Package: "test.ns", + ShortName: "rule1", + Title: "Rule 1", + }, + "test.ns.rule2": rule.Info{ + Code: "test.ns.rule2", + Package: "test.ns", + ShortName: "rule2", + Title: "Rule 2", + }, + } + + t.Run("computes successes for rules not in failures", func(t *testing.T) { + b := &basePolicyEvaluator{ + include: &Criteria{defaultItems: []string{"*"}}, + exclude: &Criteria{}, + } + + result := Outcome{ + Namespace: "test.ns", + Failures: []Result{ + { + Message: "rule1 failed", + Metadata: map[string]any{metadataCode: "test.ns.rule1"}, + }, + }, + } + + successes := b.computeSuccesses(result, rules, "", "", map[string]bool{}, nil, time.Now()) + assert.Len(t, successes, 1) + assert.Equal(t, "Pass", successes[0].Message) + assert.Equal(t, "test.ns.rule2", successes[0].Metadata[metadataCode]) + }) + + t.Run("no successes when all rules fail", func(t *testing.T) { + b := &basePolicyEvaluator{ + include: &Criteria{defaultItems: []string{"*"}}, + exclude: &Criteria{}, + } + + result := Outcome{ + Namespace: "test.ns", + Failures: []Result{ + {Message: "r1", Metadata: map[string]any{metadataCode: "test.ns.rule1"}}, + {Message: "r2", Metadata: map[string]any{metadataCode: "test.ns.rule2"}}, + }, + } + + successes := b.computeSuccesses(result, rules, "", "", map[string]bool{}, nil, time.Now()) + assert.Empty(t, successes) + }) + + t.Run("all rules succeed", func(t *testing.T) { + b := &basePolicyEvaluator{ + include: &Criteria{defaultItems: []string{"*"}}, + exclude: &Criteria{}, + } + + result := Outcome{Namespace: "test.ns"} + + successes := b.computeSuccesses(result, rules, "", "", map[string]bool{}, nil, time.Now()) + assert.Len(t, successes, 2) + }) + + t.Run("includes metadata fields", func(t *testing.T) { + extendedRules := policyRules{ + "test.ns.full": rule.Info{ + Code: "test.ns.full", + Package: "test.ns", + ShortName: "full", + Title: "Full Rule", + Description: "A complete rule", + Collections: []string{"col1"}, + DependsOn: []string{"dep1"}, + EffectiveOn: "2024-01-01T00:00:00Z", + }, + } + + b := &basePolicyEvaluator{ + include: &Criteria{defaultItems: []string{"*"}}, + exclude: &Criteria{}, + } + result := Outcome{Namespace: "test.ns"} + + successes := b.computeSuccesses(result, extendedRules, "", "", map[string]bool{}, nil, time.Now()) + require.Len(t, successes, 1) + assert.Equal(t, "Full Rule", successes[0].Metadata[metadataTitle]) + assert.Equal(t, "A complete rule", successes[0].Metadata[metadataDescription]) + assert.Equal(t, []string{"col1"}, successes[0].Metadata[metadataCollections]) + assert.Equal(t, []string{"dep1"}, successes[0].Metadata[metadataDependsOn]) + assert.Equal(t, "2024-01-01T00:00:00Z", successes[0].Metadata[metadataEffectiveOn]) + }) +} + +func TestBasePolicyEvaluatorIsResultIncluded(t *testing.T) { + t.Run("included by default with wildcard", func(t *testing.T) { + b := &basePolicyEvaluator{ + include: &Criteria{defaultItems: []string{"*"}}, + exclude: &Criteria{}, + } + + result := Result{ + Metadata: map[string]any{metadataCode: "test.rule"}, + } + + assert.True(t, b.isResultIncluded(result, "image:latest", "", map[string]bool{})) + }) + + t.Run("excluded rule", func(t *testing.T) { + b := &basePolicyEvaluator{ + include: &Criteria{}, + exclude: &Criteria{ + defaultItems: []string{"test.rule"}, + }, + } + + result := Result{ + Metadata: map[string]any{metadataCode: "test.rule"}, + } + + assert.False(t, b.isResultIncluded(result, "image:latest", "", map[string]bool{})) + }) +} + +func TestBasePolicyEvaluatorPostProcessResults(t *testing.T) { + rules := policyRules{ + "test.ns.rule1": rule.Info{ + Code: "test.ns.rule1", + Package: "test.ns", + ShortName: "rule1", + Title: "Rule 1", + }, + } + + config := &simpleConfigProvider{effectiveTime: time.Now()} + + src := ecc.Source{} + b := &basePolicyEvaluator{ + policy: config, + rules: rules, + allRules: rules, + include: &Criteria{defaultItems: []string{"*"}}, + exclude: &Criteria{}, } + b.policyResolver = NewIncludeExcludePolicyResolver(src, config) + + t.Run("processes results with successes", func(t *testing.T) { + ctx := context.Background() + runResults := []Outcome{ + { + Namespace: "test.ns", + Failures: []Result{ + { + Message: "test failure", + Metadata: map[string]any{metadataCode: "test.ns.rule1"}, + }, + }, + }, + } + target := EvaluationTarget{Target: "image:latest"} + + results, err := b.postProcessResults(ctx, runResults, target) + require.NoError(t, err) + assert.NotEmpty(t, results) + }) + + t.Run("returns error on no results", func(t *testing.T) { + ctx := context.Background() + emptyRules := policyRules{} + bEmpty := &basePolicyEvaluator{ + policy: config, + rules: emptyRules, + allRules: emptyRules, + include: &Criteria{defaultItems: []string{"*"}}, + exclude: &Criteria{}, + } + bEmpty.policyResolver = NewIncludeExcludePolicyResolver(src, config) + + runResults := []Outcome{ + {Namespace: "empty.ns"}, + } + target := EvaluationTarget{Target: "image:latest"} + + _, err := bEmpty.postProcessResults(ctx, runResults, target) + assert.Error(t, err) + assert.Contains(t, err.Error(), "no successes, warnings, or failures") + }) +} + +func TestBasePolicyEvaluatorResolveFilteredNamespaces(t *testing.T) { + t.Run("nil resolver returns nil", func(t *testing.T) { + b := &basePolicyEvaluator{} + ns := b.resolveFilteredNamespaces(EvaluationTarget{}) + assert.Nil(t, ns) + }) + + t.Run("with resolver returns packages", func(t *testing.T) { + b := &basePolicyEvaluator{ + allRules: policyRules{ + "test.ns.rule1": rule.Info{ + Code: "test.ns.rule1", + Package: "test.ns", + }, + }, + } + b.policyResolver = NewIncludeExcludePolicyResolver(ecc.Source{}, &simpleConfigProvider{}) + + ns := b.resolveFilteredNamespaces(EvaluationTarget{Target: "image:latest"}) + assert.NotNil(t, ns) + }) +} + +func TestBasePolicyEvaluatorInitPolicyResolver(t *testing.T) { + config := &simpleConfigProvider{} + src := ecc.Source{} + + b := &basePolicyEvaluator{} + b.initPolicyResolver(src, config) + + assert.NotNil(t, b.policyResolver) + assert.NotNil(t, b.include) + assert.NotNil(t, b.exclude) } diff --git a/internal/image/validate.go b/internal/image/validate.go index cf80840fa..998b9e0f6 100644 --- a/internal/image/validate.go +++ b/internal/image/validate.go @@ -112,7 +112,13 @@ func ValidateImage(ctx context.Context, comp app.SnapshotComponent, snap *app.Sn return out, nil } - inputPath, inputJSON, err := a.WriteInputFile(ctx) + inputMap, inputJSON, err := a.BuildInput(ctx) + if err != nil { + log.Debug("Problem building input!") + return nil, err + } + + inputPath, _, err := a.WriteInputFile(ctx) if err != nil { log.Debug("Problem writing input files!") return nil, err @@ -121,9 +127,9 @@ func ValidateImage(ctx context.Context, comp app.SnapshotComponent, snap *app.Sn var allResults []evaluator.Outcome for _, e := range evaluators { - // Todo maybe: Handle each one concurrently target := evaluator.EvaluationTarget{ Inputs: []string{inputPath}, + ParsedInput: inputMap, ComponentName: comp.Name, } if ref := a.ImageReference(ctx); ref == "" { @@ -133,10 +139,10 @@ func ValidateImage(ctx context.Context, comp app.SnapshotComponent, snap *app.Sn } results, err := e.Evaluate(ctx, target) - log.Debug("\n\nRunning conftest policy check\n\n") + log.Debug("\n\nRunning policy check\n\n") if err != nil { - log.Debug("Problem running conftest policy check!") + log.Debug("Problem running policy check!") return nil, err } allResults = append(allResults, results...) diff --git a/internal/validate/vsa/fallback.go b/internal/validate/vsa/fallback.go index b505eefe3..d0210e834 100644 --- a/internal/validate/vsa/fallback.go +++ b/internal/validate/vsa/fallback.go @@ -163,7 +163,8 @@ func CreateWorkerFallbackContext(ctx context.Context, fallbackPolicy policy.Poli var err error if utils.IsOpaEnabled() { log.Debugf("🔄 Worker: Using OPA evaluator") - c, err = evaluator.NewOPAEvaluator() + c, err = evaluator.NewOPAEvaluator( + ctx, policySources, fallbackPolicy, sourceGroup, nil) } else { log.Debugf("🔄 Worker: Using Conftest evaluator with filter type: include-exclude") // Use the unified filtering approach with the specified filter type