From d2396959cce495fe503a921be80431e6d2abbdf1 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Mon, 17 Aug 2026 14:56:15 +0100 Subject: [PATCH] Add native extension analysis --- README.md | 2 +- analyze.go | 51 ++++++++++++++++++++++++++++++++++++++++++------ analyze_test.go | 52 +++++++++++++++++++++++++++++++++++++++++++++++-- candidate.go | 13 ++++++++++--- go.mod | 10 ++++++++-- go.sum | 25 ++++++++++++++++++++---- 6 files changed, 135 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a744562..e32147a 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ ranked := dependents.Rank(kept, 10, nil) `DefaultScore` favors source references and tests after checkout analysis. Callers can pass another `ScoreFunc` to `Rank`, so security exposure and contract-test selection can use different policies. -`Analyze` accepts any `Checkout`. `CloneCheckout` uses a direct checkout and supports full history for Hyrum, while `CacheCheckout` copies from a persistent `git-pkgs/clone` cache for Scrutineer. A caller that supplies `Workdir` or sets `Keep` receives the checkout path on each analyzed candidate for follow-up work. +`Analyze` accepts any `Checkout`. `CloneCheckout` uses a direct checkout and supports full history for Hyrum, while `CacheCheckout` copies from a persistent `git-pkgs/clone` cache for Scrutineer. A caller that supplies `Workdir` or sets `Keep` receives the checkout path on each analyzed candidate for follow-up work. Set `DetectNativeExtensions` to record native-extension toolchains and their build commands, including Maturin, napi-rs, Neon, rb-sys, Rustler, and setuptools-rust. After analysis, `FilterOptions.RequireTests` and `RequireImports` reproduce the contract-test eligibility used by downstream. Scrutineer can require upstream references without excluding repositories that have no conventional test files. diff --git a/analyze.go b/analyze.go index 317c2a8..9709fe7 100644 --- a/analyze.go +++ b/analyze.go @@ -9,10 +9,12 @@ import ( "io/fs" "os" "path/filepath" + "sort" "strings" "sync" "github.com/git-pkgs/brief" + "github.com/git-pkgs/brief/detect" "github.com/git-pkgs/brief/kb" ) @@ -23,10 +25,11 @@ const ( // AnalyzeOptions controls checkout analysis. type AnalyzeOptions struct { - Upstreams []string - Workdir string - Checkout Checkout - Keep bool + Upstreams []string + Workdir string + Checkout Checkout + Keep bool + DetectNativeExtensions bool } // AnalysisFailure records one candidate that could not be checked out or @@ -85,6 +88,13 @@ func Analyze(ctx context.Context, candidates []Candidate, opts AnalyzeOptions) ( result.Failures = append(result.Failures, AnalysisFailure{Repository: candidate.Repository, Err: err}) continue } + if opts.DetectNativeExtensions { + analysis.NativeExtensions, err = detectNativeExtensions(destination) + if err != nil { + result.Failures = append(result.Failures, AnalysisFailure{Repository: candidate.Repository, Err: err}) + continue + } + } candidate.Analysis = analysis candidate.Analyzed = true candidate.Commit = commit @@ -150,9 +160,13 @@ func candidateDirectory(repository string) string { return hex.EncodeToString(sum[:]) } +var loadKnowledge = sync.OnceValues(func() (*kb.KnowledgeBase, error) { + return kb.Load(brief.KnowledgeFS) +}) + var testDirs = sync.OnceValue(func() map[string]bool { - dirs := map[string]bool{} - knowledge, err := kb.Load(brief.KnowledgeFS) + dirs := make(map[string]bool) + knowledge, err := loadKnowledge() if err != nil { return dirs } @@ -162,6 +176,31 @@ var testDirs = sync.OnceValue(func() map[string]bool { return dirs }) +func detectNativeExtensions(root string) ([]NativeExtension, error) { + knowledge, err := loadKnowledge() + if err != nil { + return nil, err + } + report, err := detect.New(knowledge, root).Run() + if err != nil { + return nil, err + } + + detections := report.Tools["native_extension"] + extensions := make([]NativeExtension, 0, len(detections)) + for _, detection := range detections { + extension := NativeExtension{Name: detection.Name} + if detection.Command != nil { + extension.BuildCommand = detection.Command.Run + } + extensions = append(extensions, extension) + } + sort.Slice(extensions, func(i, j int) bool { + return extensions[i].Name < extensions[j].Name + }) + return extensions, nil +} + func isTestFile(base string) bool { stem, extension, ok := strings.Cut(base, ".") if !ok { diff --git a/analyze_test.go b/analyze_test.go index 4b3c42d..e9df678 100644 --- a/analyze_test.go +++ b/analyze_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "reflect" "strings" "testing" ) @@ -71,7 +72,7 @@ func TestAnalyzeDirectorySkipsSymlinks(t *testing.T) { if err != nil { t.Fatalf("AnalyzeDirectory: %v", err) } - if got != (Analysis{}) { + if !reflect.DeepEqual(got, Analysis{}) { t.Errorf("analysis = %+v, want symlink excluded", got) } } @@ -105,7 +106,7 @@ func TestAnalyzeKeepsFailuresAndPersistentDirectories(t *testing.T) { t.Fatalf("result = %+v", result) } good := result.Candidates[0] - if !good.Analyzed || good.Commit != "abc123" || good.Analysis != (Analysis{TestFiles: 1, ImportFiles: 1}) { + if !good.Analyzed || good.Commit != "abc123" || !reflect.DeepEqual(good.Analysis, Analysis{TestFiles: 1, ImportFiles: 1}) { t.Errorf("good candidate = %+v", good) } if good.Directory == "" { @@ -125,6 +126,53 @@ func TestAnalyzeKeepsFailuresAndPersistentDirectories(t *testing.T) { } } +func TestAnalyzeDetectsNativeExtensions(t *testing.T) { + checkout := CheckoutFunc(func(_ context.Context, _ string, destination string) (string, error) { + writeTree(t, destination, map[string]string{ + "Cargo.toml": `[package] +name = "native-package" +version = "0.1.0" + +[dependencies] +rb-sys = "0.9" +`, + "Gemfile": `source "https://rubygems.org" +gem "rb_sys" +`, + "pyproject.toml": `[build-system] +requires = ["maturin>=1.0,<2.0"] +build-backend = "maturin" + +[project] +name = "native-package" +version = "0.1.0" + +[tool.maturin] +bindings = "pyo3" +`, + }) + return "abc123", nil + }) + + result, err := Analyze(context.Background(), []Candidate{{Repository: "https://example.com/native"}}, AnalyzeOptions{ + Checkout: checkout, + DetectNativeExtensions: true, + }) + if err != nil { + t.Fatalf("Analyze: %v", err) + } + if len(result.Failures) != 0 { + t.Fatalf("failures = %+v", result.Failures) + } + want := []NativeExtension{ + {Name: "Maturin", BuildCommand: "maturin develop"}, + {Name: "rb-sys", BuildCommand: "bundle exec rake compile"}, + } + if got := result.Candidates[0].Analysis.NativeExtensions; !reflect.DeepEqual(got, want) { + t.Errorf("native extensions = %+v, want %+v", got, want) + } +} + func TestAnalyzeStopsOnCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() diff --git a/candidate.go b/candidate.go index 6d6765d..986a2cf 100644 --- a/candidate.go +++ b/candidate.go @@ -68,10 +68,17 @@ type Relationship struct { Dependent PackageRef } -// Analysis contains checkout-derived ranking signals. +// NativeExtension describes a detected native-extension toolchain. +type NativeExtension struct { + Name string + BuildCommand string +} + +// Analysis contains checkout-derived ranking signals and integrations. type Analysis struct { - TestFiles int - ImportFiles int + TestFiles int + ImportFiles int + NativeExtensions []NativeExtension } // Candidate is one repository containing packages that depend on one or more diff --git a/go.mod b/go.mod index f80c473..6552ba3 100644 --- a/go.mod +++ b/go.mod @@ -3,16 +3,20 @@ module github.com/git-pkgs/dependents go 1.26 require ( - github.com/git-pkgs/brief v0.11.0 - github.com/git-pkgs/clone v0.5.0 + github.com/git-pkgs/brief v0.12.1 + github.com/git-pkgs/clone v0.7.1 github.com/git-pkgs/enrichment v0.7.0 ) require ( github.com/BurntSushi/toml v1.6.0 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect + github.com/bazelbuild/buildtools v0.0.0-20260716142318-04cf7de1434f // indirect + github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect github.com/ecosyste-ms/ecosystems-go v0.4.0 // indirect + github.com/git-pkgs/licensecheck v0.4.1 // indirect github.com/git-pkgs/magic v0.2.0 // indirect + github.com/git-pkgs/manifests v0.10.0 // indirect github.com/git-pkgs/packageurl-go v0.3.1 // indirect github.com/git-pkgs/pom v0.1.7 // indirect github.com/git-pkgs/purl v0.1.17 // indirect @@ -26,4 +30,6 @@ require ( github.com/oapi-codegen/runtime v1.6.0 // indirect github.com/package-url/packageurl-go v0.1.6 // indirect github.com/pandatix/go-cvss v0.6.2 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index f91e25c..5790af1 100644 --- a/go.sum +++ b/go.sum @@ -3,20 +3,28 @@ github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2 github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= +github.com/bazelbuild/buildtools v0.0.0-20260716142318-04cf7de1434f h1:2mT6QcXmMFtvg7bezs1Fef7nnpJyeCmfWoWNjwtvNZ4= +github.com/bazelbuild/buildtools v0.0.0-20260716142318-04cf7de1434f/go.mod h1:PLNUetjLa77TCCziPsz0EI8a6CUxgC+1jgmWv0H25tg= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs= +github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/ecosyste-ms/ecosystems-go v0.4.0 h1:5A+zF+XWT8sYYYjlc2/tI1SmiDGzbHLyT9CapVc5dGA= github.com/ecosyste-ms/ecosystems-go v0.4.0/go.mod h1:FVswCrp3DQkur1HjVqfDF/gYrDSEmiFflntcB1G0DbA= -github.com/git-pkgs/brief v0.11.0 h1:rTzNw+46oVdxPpfMKJjwjqQ81Th2oAwm0VFaZCwBr+s= -github.com/git-pkgs/brief v0.11.0/go.mod h1:BOtq1uUQ5YEgvOCBZycrT2ROO9EgsIjNM82MhAb0WH8= -github.com/git-pkgs/clone v0.5.0 h1:0cYpr8PQpylqtvA9v1NBE9JXJeLLlrspJbvl2JYVLxU= -github.com/git-pkgs/clone v0.5.0/go.mod h1:nixjE44maKRXbu/dEiunJmHApWgLYp0IR7J2yWaYHv8= +github.com/git-pkgs/brief v0.12.1 h1:s7NmyUPnJHIOe654TKNBWXBjhhju3d8zlW8l3sBj0rA= +github.com/git-pkgs/brief v0.12.1/go.mod h1:oUNxWSJwJGMcu3JFsN/giySt3hzWgRvM9S+XnluhhFM= +github.com/git-pkgs/clone v0.7.1 h1:ai1sI2EypbgJemWEwSIW7v+SndnHHusGxOKD4+bLT2Y= +github.com/git-pkgs/clone v0.7.1/go.mod h1:Dk6k+HGvI8+OR27ijxGtCcHhSIyhGBWShE6S6ie5v0s= github.com/git-pkgs/enrichment v0.7.0 h1:LfIzlVArc2p0MONO08ybC5jiHlysfzS3YyZ5ry2d6Lw= github.com/git-pkgs/enrichment v0.7.0/go.mod h1:ZgZJq7cz1H/nlkgHjCdIRLF/TA5VcbUmpE970VQmG14= +github.com/git-pkgs/licensecheck v0.4.1 h1:b5ilmpIpgeeewBFjdhJ4W7jvwPIFsYQ7ujZma7sli6k= +github.com/git-pkgs/licensecheck v0.4.1/go.mod h1:cfFO7yHHPeuXsoODHBWyevajH2yWcbfkIFELyG3ZpU0= github.com/git-pkgs/magic v0.2.0 h1:c7HqVxnP8c88EaVMH0/KraDFVTcmiXckRiSvNZEnvMQ= github.com/git-pkgs/magic v0.2.0/go.mod h1:3ndidt+yvFaI1M0aEkkzkOlFnLPkeVQASIUojazcxCI= +github.com/git-pkgs/manifests v0.10.0 h1:wRYoX8l5mdTxSQ2/99gIByLQPBJUPW2VdrBSX0If2fA= +github.com/git-pkgs/manifests v0.10.0/go.mod h1:y1p9ICibE2tqveiHJEJZz4mayo+mt6MhaBY3LHHr1mY= github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M= github.com/git-pkgs/packageurl-go v0.3.1/go.mod h1:rcIxiG37BlQLB6FZfgdj9Fm7yjhRQd3l+5o7J0QPAk4= github.com/git-pkgs/pom v0.1.7 h1:4yKdtw6eyShtjul6bcZdyz7yLQ+jdrYeYkKbskDGi4c= @@ -36,6 +44,10 @@ github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWo github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/oapi-codegen/nullable v1.2.0 h1:VflFkDW980KhBPiFF7nWSyjg+r4Obqj8lXipV0UkP5w= github.com/oapi-codegen/nullable v1.2.0/go.mod h1:KUZ3vUzkmEKY90ksAmit2+5juDIhIZhfDl+0PwOQlFY= github.com/oapi-codegen/runtime v1.6.0 h1:7Xx+GlueD6nRuyKoCPzL434Jfi3BetbiJOrzCHp/VPU= @@ -46,10 +58,15 @@ github.com/pandatix/go-cvss v0.6.2 h1:TFiHlzUkT67s6UkelHmK6s1INKVUG7nlKYiWWDTITG github.com/pandatix/go-cvss v0.6.2/go.mod h1:jDXYlQBZrc8nvrMUVVvTG8PhmuShOnKrxP53nOFkt8Q= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=