diff --git a/README.md b/README.md index e76d3ba..b02fdc1 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ func main() { | nix | flake.nix | flake.lock, sources.json | | pre-commit | .pre-commit-config.yaml, prek.toml | | | npm | package.json | package-lock.json, npm-shrinkwrap.json, yarn.lock, pnpm-lock.yaml, bun.lock, npm-ls.json | -| nuget | *.csproj, *.vbproj, *.fsproj, *.nuspec, packages.config, Project.json | packages.lock.json, paket.lock, project.assets.json, *.deps.json, Project.lock.json | +| nuget | *.csproj, *.vbproj, *.fsproj, *.nuspec, packages.config, Directory.Packages.props, Project.json | packages.lock.json, paket.lock, project.assets.json, *.deps.json, Project.lock.json | | opam | opam, *.opam | | | pub | pubspec.yaml | pubspec.lock | | pypi | requirements.txt, Pipfile, pyproject.toml, setup.py, setup.cfg | Pipfile.lock, poetry.lock, pdm.lock, uv.lock, pip-dependency-graph.json, pip-resolved-dependencies.txt, pylock.toml | @@ -254,6 +254,7 @@ type Declaration struct { Name string // Package name Version string // Version requirement as written in the manifest Scope Scope // runtime, development, test, build, optional + Direct bool // Direct rather than generated or transitive PURL string // Versionless Package URL Location string // Opaque parser-defined identity within the manifest } @@ -263,14 +264,21 @@ Declarations preserve source-level references without applying inheritance, merging, interpolation, or other effective-model resolution. Consumers can use `Location` to match the same logical entry across edits, but should not parse its ecosystem-specific value. A declaration PURL omits the version because the -raw requirement may be a range or property expression. +raw requirement may be a range or property expression. When a parser supplies +its own PURL, `Parse` preserves it so one manifest can refer to packages from +different ecosystems. Otherwise `Parse` builds the PURL from the parser's +ecosystem. + +`Direct` distinguishes explicit requirements from generated or transitive +entries when the source format records that distinction, such as `go.mod`. Parsers that do not preserve source locations leave `Declarations` empty. -Declarations are available for `package.json`, Python requirements files, -`pyproject.toml`, GitHub Actions workflows, and `pom.xml`. The Maven parser -includes parents, dependencies, dependency management, plugins, plugin -dependencies, plugin management, build extensions, and their profile-scoped -forms. +Declarations are available for `package.json`, Cargo manifests, `go.mod`, Python +requirements files, `pyproject.toml`, GitHub Actions workflows, `gleam.toml`, +`pom.xml`, NuGet project and package files, and +`Directory.Packages.props`. The Maven parser includes parents, +dependencies, dependency management, plugins, plugin dependencies, plugin +management, build extensions, and their profile-scoped forms. ### ParseResult diff --git a/internal/cargo/cargo.go b/internal/cargo/cargo.go index 4e46976..89d3e00 100644 --- a/internal/cargo/cargo.go +++ b/internal/cargo/cargo.go @@ -1,6 +1,7 @@ package cargo import ( + "net/url" "strings" "github.com/BurntSushi/toml" @@ -29,6 +30,14 @@ func (p *cargoTomlParser) Parse(filename string, content []byte) (*core.Result, Dependencies map[string]any `toml:"dependencies"` DevDependencies map[string]any `toml:"dev-dependencies"` BuildDependencies map[string]any `toml:"build-dependencies"` + Target map[string]struct { + Dependencies map[string]any `toml:"dependencies"` + DevDependencies map[string]any `toml:"dev-dependencies"` + BuildDependencies map[string]any `toml:"build-dependencies"` + } `toml:"target"` + Workspace struct { + Dependencies map[string]any `toml:"dependencies"` + } `toml:"workspace"` } if _, err := toml.Decode(string(content), &cargo); err != nil { @@ -36,47 +45,22 @@ func (p *cargoTomlParser) Parse(filename string, content []byte) (*core.Result, } var deps []core.Dependency + var declarations []core.Declaration pkgName := cargo.Package.Name - for name, value := range cargo.Dependencies { - version := extractCargoVersion(value) - // Skip local path dependencies - if isLocalCargoDep(value) { - continue - } - deps = append(deps, core.Dependency{ - Name: name, - Version: version, - Scope: core.Runtime, - Direct: true, - }) - } - - for name, value := range cargo.DevDependencies { - version := extractCargoVersion(value) - if isLocalCargoDep(value) { - continue - } - deps = append(deps, core.Dependency{ - Name: name, - Version: version, - Scope: core.Development, - Direct: true, - }) - } - - for name, value := range cargo.BuildDependencies { - version := extractCargoVersion(value) - if isLocalCargoDep(value) { - continue - } - deps = append(deps, core.Dependency{ - Name: name, - Version: version, - Scope: core.Build, - Direct: true, - }) + collectCargoDependencies(&deps, cargo.Dependencies, core.Runtime) + collectCargoDependencies(&deps, cargo.DevDependencies, core.Development) + collectCargoDependencies(&deps, cargo.BuildDependencies, core.Build) + collectCargoDeclarations(&declarations, "dependencies", cargo.Dependencies, core.Runtime, pkgName) + collectCargoDeclarations(&declarations, "dev-dependencies", cargo.DevDependencies, core.Development, pkgName) + collectCargoDeclarations(&declarations, "build-dependencies", cargo.BuildDependencies, core.Build, pkgName) + for target, groups := range cargo.Target { + prefix := "target/" + url.PathEscape(target) + "/" + collectCargoDeclarations(&declarations, prefix+"dependencies", groups.Dependencies, core.Runtime, pkgName) + collectCargoDeclarations(&declarations, prefix+"dev-dependencies", groups.DevDependencies, core.Development, pkgName) + collectCargoDeclarations(&declarations, prefix+"build-dependencies", groups.BuildDependencies, core.Build, pkgName) } + collectCargoDeclarations(&declarations, "workspace/dependencies", cargo.Workspace.Dependencies, core.Runtime, pkgName) // Filter out self-reference filtered := deps[:0] @@ -96,9 +80,69 @@ func (p *cargoTomlParser) Parse(filename string, content []byte) (*core.Result, Licenses: licenses, LicenseFile: cargo.Package.LicenseFile, Dependencies: filtered, + Declarations: declarations, }, nil } +// collectCargoDependencies appends the dependency inventory from one Cargo +// dependency table. +func collectCargoDependencies(dependencies *[]core.Dependency, values map[string]any, scope core.Scope) { + for name, value := range values { + if isLocalCargoDep(value) { + continue + } + *dependencies = append(*dependencies, core.Dependency{ + Name: name, + Version: extractCargoVersion(value), + Scope: scope, + Direct: true, + }) + } +} + +// collectCargoDeclarations appends source declarations from one Cargo +// dependency table. +func collectCargoDeclarations( + declarations *[]core.Declaration, + prefix string, + values map[string]any, + scope core.Scope, + selfName string, +) { + for name, value := range values { + version := extractCargoVersion(value) + declaredName, ok := cargoRegistryDeclaration(name, value) + if !ok || declaredName == selfName { + continue + } + *declarations = append(*declarations, core.Declaration{ + Name: declaredName, + Version: version, + Scope: scope, + Direct: true, + Location: prefix + "/" + url.PathEscape(name), + }) + } +} + +// cargoRegistryDeclaration returns the registry package name for a dependency. +// Local, git, workspace and named-registry sources are not registry-checkable. +func cargoRegistryDeclaration(name string, value any) (string, bool) { + properties, ok := value.(map[string]any) + if !ok { + return name, true + } + for _, source := range []string{"git", "path", "registry", "workspace"} { + if _, found := properties[source]; found { + return "", false + } + } + if packageName, ok := properties["package"].(string); ok && packageName != "" { + return packageName, true + } + return name, true +} + func extractCargoVersion(value any) string { switch v := value.(type) { case string: diff --git a/internal/cargo/cargo_test.go b/internal/cargo/cargo_test.go index c2829f3..5dd4d0e 100644 --- a/internal/cargo/cargo_test.go +++ b/internal/cargo/cargo_test.go @@ -58,6 +58,71 @@ func TestCargoToml(t *testing.T) { } } +func TestCargoTomlDeclarations(t *testing.T) { + content := []byte(`[package] +name = "application" + +[dependencies] +serde = "=1.0.0" +alias = { package = "actual", version = "=2.0.0" } +local = { path = "../local", version = "=3.0.0" } +git-dep = { git = "https://example.com/repo.git", version = "=4.0.0" } +private = { registry = "private", version = "=5.0.0" } + +[dev-dependencies] +serde = "=1.1.0" + +[build-dependencies] +cc = "=1.2.0" + +[target.'cfg(unix)'.dependencies] +libc = "=0.2.0" + +[workspace.dependencies] +anyhow = "=1.0.0" +`) + + parser := &cargoTomlParser{} + result, err := parser.Parse("Cargo.toml", content) + if err != nil { + t.Fatalf("Parse failed: %v", err) + } + + want := map[string]struct { + name string + version string + scope core.Scope + }{ + "dependencies/serde": {"serde", "=1.0.0", core.Runtime}, + "dependencies/alias": {"actual", "=2.0.0", core.Runtime}, + "dev-dependencies/serde": {"serde", "=1.1.0", core.Development}, + "build-dependencies/cc": {"cc", "=1.2.0", core.Build}, + "target/cfg%28unix%29/dependencies/libc": {"libc", "=0.2.0", core.Runtime}, + "workspace/dependencies/anyhow": {"anyhow", "=1.0.0", core.Runtime}, + } + if len(result.Declarations) != len(want) { + t.Fatalf("Declarations has %d entries, want %d: %+v", len(result.Declarations), len(want), result.Declarations) + } + for _, declaration := range result.Declarations { + expected, ok := want[declaration.Location] + if !ok { + t.Errorf("unexpected declaration: %+v", declaration) + continue + } + if declaration.Name != expected.name || declaration.Version != expected.version || declaration.Scope != expected.scope { + t.Errorf("declaration at %q = %+v, want %+v", declaration.Location, declaration, expected) + } + } + if len(result.Dependencies) != 6 { + t.Fatalf("Dependencies has %d entries, want 6: %+v", len(result.Dependencies), result.Dependencies) + } + for _, dependency := range result.Dependencies { + if dependency.Name == "libc" || dependency.Name == "anyhow" { + t.Errorf("target and workspace declarations should not widen Dependencies: %+v", dependency) + } + } +} + func TestCargoLock(t *testing.T) { content, err := os.ReadFile("../../testdata/cargo/Cargo.lock") if err != nil { diff --git a/internal/core/helpers.go b/internal/core/helpers.go index cead20c..e5b71dc 100644 --- a/internal/core/helpers.go +++ b/internal/core/helpers.go @@ -1,6 +1,19 @@ package core -import "strings" +import ( + "strconv" + "strings" +) + +// NextLocation returns base with a numeric suffix when seen already contains +// it, so repeated declarations at the same logical position stay addressable. +func NextLocation(seen map[string]int, base string) string { + seen[base]++ + if seen[base] > 1 { + return base + "/" + strconv.Itoa(seen[base]) + } + return base +} // ForEachLine iterates over lines in content without allocating a slice. func ForEachLine(content string, fn func(line string) bool) { diff --git a/internal/core/types.go b/internal/core/types.go index 075ab79..e4617f8 100644 --- a/internal/core/types.go +++ b/internal/core/types.go @@ -40,9 +40,14 @@ type Dependency struct { // before effective-model resolution or inheritance. Location is // ecosystem-specific and should be treated as an opaque identity. type Declaration struct { - Name string - Version string - Scope Scope + Name string + Version string + Scope Scope + // Direct reports whether the manifest declares the package as a direct + // dependency rather than a generated or transitive requirement. + Direct bool + // PURL identifies the declared package without a version. Parsers may set + // it when a file can contain references from more than one ecosystem. PURL string Location string } diff --git a/internal/github_actions/github_actions.go b/internal/github_actions/github_actions.go index 9a3df27..41891bb 100644 --- a/internal/github_actions/github_actions.go +++ b/internal/github_actions/github_actions.go @@ -3,7 +3,6 @@ package github_actions import ( "net/url" "path/filepath" - "strconv" "strings" "github.com/git-pkgs/manifests/internal/core" @@ -95,16 +94,12 @@ func collectStepActions( continue } if !strings.HasPrefix(name, "docker://") { - base := "jobs/" + url.PathEscape(jobName) + "/steps/" + url.PathEscape(name) - locations[base]++ - location := base - if locations[base] > 1 { - location += "/" + strconv.Itoa(locations[base]) - } + location := core.NextLocation(locations, "jobs/"+url.PathEscape(jobName)+"/steps/"+url.PathEscape(name)) *declarations = append(*declarations, core.Declaration{ Name: name, Version: version, Scope: core.Runtime, + Direct: true, Location: location, }) } diff --git a/internal/gleam/gleam.go b/internal/gleam/gleam.go index c5f73d8..ca59467 100644 --- a/internal/gleam/gleam.go +++ b/internal/gleam/gleam.go @@ -1,6 +1,8 @@ package gleam import ( + "net/url" + "github.com/BurntSushi/toml" "github.com/git-pkgs/manifests/internal/core" ) @@ -13,10 +15,10 @@ func init() { type gleamTomlParser struct{} type gleamToml struct { - Name string `toml:"name"` - Version string `toml:"version"` - Dependencies map[string]string `toml:"dependencies"` - DevDependencies map[string]string `toml:"dev-dependencies"` + Name string `toml:"name"` + Version string `toml:"version"` + Dependencies map[string]any `toml:"dependencies"` + DevDependencies map[string]any `toml:"dev-dependencies"` } func (p *gleamTomlParser) Parse(filename string, content []byte) (*core.Result, error) { @@ -26,24 +28,52 @@ func (p *gleamTomlParser) Parse(filename string, content []byte) (*core.Result, } var deps []core.Dependency + var declarations []core.Declaration - for name, version := range gleam.Dependencies { + for name, value := range gleam.Dependencies { + version, ok := value.(string) + if !ok { + continue + } deps = append(deps, core.Dependency{ Name: name, Version: version, Scope: core.Runtime, Direct: true, }) + declarations = append(declarations, core.Declaration{ + Name: name, + Version: version, + Scope: core.Runtime, + Direct: true, + Location: "dependencies/" + url.PathEscape(name), + }) } - for name, version := range gleam.DevDependencies { + for name, value := range gleam.DevDependencies { + version, ok := value.(string) + if !ok { + continue + } deps = append(deps, core.Dependency{ Name: name, Version: version, Scope: core.Development, Direct: true, }) + declarations = append(declarations, core.Declaration{ + Name: name, + Version: version, + Scope: core.Development, + Direct: true, + Location: "dev-dependencies/" + url.PathEscape(name), + }) } - return &core.Result{Name: gleam.Name, Version: gleam.Version, Dependencies: deps}, nil + return &core.Result{ + Name: gleam.Name, + Version: gleam.Version, + Dependencies: deps, + Declarations: declarations, + }, nil } diff --git a/internal/gleam/gleam_test.go b/internal/gleam/gleam_test.go index 5a78950..67b43dd 100644 --- a/internal/gleam/gleam_test.go +++ b/internal/gleam/gleam_test.go @@ -52,4 +52,53 @@ func TestGleamToml(t *testing.T) { t.Errorf("%s scope = %v, want %v", exp.name, dep.Scope, exp.scope) } } + + wantDeclarations := map[string]core.Scope{ + "dependencies/gleam_stdlib": core.Runtime, + "dependencies/gleam_http": core.Runtime, + "dev-dependencies/gleeunit": core.Development, + } + if len(res.Declarations) != len(wantDeclarations) { + t.Fatalf("Declarations has %d entries, want %d: %+v", len(res.Declarations), len(wantDeclarations), res.Declarations) + } + for _, declaration := range res.Declarations { + if scope, ok := wantDeclarations[declaration.Location]; !ok || declaration.Scope != scope || !declaration.Direct { + t.Errorf("unexpected declaration: %+v", declaration) + } + } +} + +func TestGleamTomlDeclarationsIgnoreNonHexSources(t *testing.T) { + content := []byte(`name = "example" +version = "1.0.0" + +[dependencies] +gleam_stdlib = ">= 1.0.0 and < 2.0.0" +local_package = { path = "../local_package" } +git_package = { git = "https://example.com/git_package.git", ref = "main" } + +[dev-dependencies] +gleeunit = ">= 1.0.0 and < 2.0.0" +`) + + result, err := (&gleamTomlParser{}).Parse("gleam.toml", content) + if err != nil { + t.Fatalf("Parse failed: %v", err) + } + + want := map[string]core.Scope{ + "dependencies/gleam_stdlib": core.Runtime, + "dev-dependencies/gleeunit": core.Development, + } + if len(result.Declarations) != len(want) { + t.Fatalf("Declarations has %d entries, want %d: %+v", len(result.Declarations), len(want), result.Declarations) + } + for _, declaration := range result.Declarations { + if scope, ok := want[declaration.Location]; !ok || declaration.Scope != scope || !declaration.Direct { + t.Errorf("unexpected declaration: %+v", declaration) + } + } + if len(result.Dependencies) != len(want) { + t.Fatalf("Dependencies has %d entries, want %d: %+v", len(result.Dependencies), len(want), result.Dependencies) + } } diff --git a/internal/golang/golang.go b/internal/golang/golang.go index 5d96b8b..26e8526 100644 --- a/internal/golang/golang.go +++ b/internal/golang/golang.go @@ -1,9 +1,11 @@ package golang import ( - "github.com/git-pkgs/manifests/internal/core" + "net/url" "regexp" "strings" + + "github.com/git-pkgs/manifests/internal/core" ) func init() { @@ -20,6 +22,11 @@ func init() { // goModParser parses go.mod files. type goModParser struct{} +type moduleVersion struct { + path string + version string +} + var ( // Single-line require: require example.com/pkg v1.2.3 singleRequireRegex = regexp.MustCompile(`^\s*require\s+(\S+)\s+(\S+)`) @@ -37,7 +44,8 @@ var ( func (p *goModParser) Parse(filename string, content []byte) (*core.Result, error) { lines := strings.Split(string(content), "\n") tools := collectToolPaths(lines) - deps := collectRequireDeps(lines, tools) + replaced := collectReplacedModules(lines) + deps, declarations := collectRequireDeps(lines, tools, replaced) var modulePath string for _, line := range lines { @@ -48,7 +56,7 @@ func (p *goModParser) Parse(filename string, content []byte) (*core.Result, erro } } - return &core.Result{Name: modulePath, Dependencies: deps}, nil + return &core.Result{Name: modulePath, Dependencies: deps, Declarations: declarations}, nil } // collectToolPaths scans go.mod lines for tool directives (both single-line and block form) @@ -93,9 +101,11 @@ func collectToolPaths(lines []string) map[string]bool { // collectRequireDeps scans go.mod lines for require directives (both single-line and block form) // and returns dependencies, marking tool-related modules as development scope. -func collectRequireDeps(lines []string, tools map[string]bool) []core.Dependency { +func collectRequireDeps(lines []string, tools map[string]bool, replaced map[moduleVersion]bool) ([]core.Dependency, []core.Declaration) { var deps []core.Dependency + var declarations []core.Declaration inRequireBlock := false + locations := make(map[string]int) for _, line := range lines { trimmed := strings.TrimSpace(line) @@ -116,19 +126,86 @@ func collectRequireDeps(lines []string, tools map[string]bool) []core.Dependency if strings.HasPrefix(trimmed, "require ") && !strings.Contains(trimmed, "(") { if match := singleRequireRegex.FindStringSubmatch(trimmed); match != nil { - deps = append(deps, newRequireDep(match[1], match[2], line, tools)) + dep := newRequireDep(match[1], match[2], line, tools) + deps = append(deps, dep) + appendGoDeclaration(&declarations, locations, dep, replaced) } continue } if inRequireBlock { if match := requireEntryRegex.FindStringSubmatch(trimmed); match != nil { - deps = append(deps, newRequireDep(match[1], match[2], line, tools)) + dep := newRequireDep(match[1], match[2], line, tools) + deps = append(deps, dep) + appendGoDeclaration(&declarations, locations, dep, replaced) } } } - return deps + return deps, declarations +} + +// appendGoDeclaration records a require directive unless a replace directive +// changes that module's source. +func appendGoDeclaration( + declarations *[]core.Declaration, + locations map[string]int, + dependency core.Dependency, + replaced map[moduleVersion]bool, +) { + if replaced[moduleVersion{path: dependency.Name}] || + replaced[moduleVersion{path: dependency.Name, version: dependency.Version}] { + return + } + location := core.NextLocation(locations, "require/"+url.PathEscape(dependency.Name)) + *declarations = append(*declarations, core.Declaration{ + Name: dependency.Name, + Version: dependency.Version, + Scope: dependency.Scope, + Direct: dependency.Direct, + Location: location, + }) +} + +// collectReplacedModules returns module paths named on the left side of a +// replace directive, in either single-line or block form. +func collectReplacedModules(lines []string) map[moduleVersion]bool { + replaced := make(map[moduleVersion]bool) + inReplaceBlock := false + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "//") { + continue + } + if strings.HasPrefix(trimmed, "replace (") { + inReplaceBlock = true + continue + } + if inReplaceBlock && trimmed == ")" { + inReplaceBlock = false + continue + } + + spec := "" + if strings.HasPrefix(trimmed, "replace ") && !strings.Contains(trimmed, "(") { + spec = strings.TrimSpace(strings.TrimPrefix(trimmed, "replace ")) + } else if inReplaceBlock { + spec = trimmed + } + left, _, ok := strings.Cut(spec, "=>") + if !ok { + continue + } + fields := strings.Fields(left) + if len(fields) > 0 { + module := moduleVersion{path: fields[0]} + if len(fields) > 1 { + module.version = fields[1] + } + replaced[module] = true + } + } + return replaced } // newRequireDep builds a Dependency from a parsed require entry, determining diff --git a/internal/golang/golang_test.go b/internal/golang/golang_test.go index 535ed89..13c42d2 100644 --- a/internal/golang/golang_test.go +++ b/internal/golang/golang_test.go @@ -78,6 +78,64 @@ func TestGoMod(t *testing.T) { } } +func TestGoModDeclarations(t *testing.T) { + content := []byte(`module example.com/application + +go 1.26 + +require example.com/single v1.0.0 + +require ( + example.com/direct v2.0.0 + example.com/indirect v3.0.0 // indirect + example.com/replaced v4.0.0 + example.com/version-replaced v5.0.0 + example.com/other-version v5.0.0 + example.com/tool v6.0.0 +) + +tool example.com/tool/cmd/tool + +replace example.com/replaced => ../replaced + +replace ( + example.com/version-replaced v5.0.0 => example.com/fork v5.0.1 + example.com/other-version v4.0.0 => example.com/fork v4.0.1 +) +`) + + parser := &goModParser{} + result, err := parser.Parse("go.mod", content) + if err != nil { + t.Fatalf("Parse failed: %v", err) + } + + want := map[string]struct { + name string + scope core.Scope + direct bool + }{ + "require/example.com%2Fsingle": {"example.com/single", core.Runtime, true}, + "require/example.com%2Fdirect": {"example.com/direct", core.Runtime, true}, + "require/example.com%2Findirect": {"example.com/indirect", core.Runtime, false}, + "require/example.com%2Fother-version": {"example.com/other-version", core.Runtime, true}, + "require/example.com%2Ftool": {"example.com/tool", core.Development, true}, + } + if len(result.Declarations) != len(want) { + t.Fatalf("Declarations has %d entries, want %d: %+v", len(result.Declarations), len(want), result.Declarations) + } + for _, declaration := range result.Declarations { + expected, ok := want[declaration.Location] + if !ok { + t.Errorf("unexpected declaration: %+v", declaration) + continue + } + if declaration.Name != expected.name || declaration.Scope != expected.scope || declaration.Direct != expected.direct { + t.Errorf("declaration at %q = %+v, want %+v", declaration.Location, declaration, expected) + } + } +} + func TestGoSum(t *testing.T) { content, err := os.ReadFile("../../testdata/golang/go.sum") if err != nil { diff --git a/internal/maven/declarations.go b/internal/maven/declarations.go index 2d93578..80d4e74 100644 --- a/internal/maven/declarations.go +++ b/internal/maven/declarations.go @@ -139,6 +139,7 @@ func appendMavenDeclaration( Name: name, Version: strings.TrimSpace(version), Scope: scope, + Direct: true, Location: location + "/" + url.PathEscape(key), }) return name diff --git a/internal/npm/npm.go b/internal/npm/npm.go index 0a46c6f..3ebef54 100644 --- a/internal/npm/npm.go +++ b/internal/npm/npm.go @@ -88,6 +88,7 @@ func collectNpmDeclarations( Name: realName, Version: realVersion, Scope: scope, + Direct: true, Location: location + "/" + url.PathEscape(name), }) } diff --git a/internal/nuget/nuget.go b/internal/nuget/nuget.go index a7d7398..921ac08 100644 --- a/internal/nuget/nuget.go +++ b/internal/nuget/nuget.go @@ -3,10 +3,12 @@ package nuget import ( "encoding/json" "encoding/xml" - "github.com/git-pkgs/manifests/internal/core" + "net/url" "path/filepath" "regexp" "strings" + + "github.com/git-pkgs/manifests/internal/core" ) func init() { @@ -15,6 +17,7 @@ func init() { core.Register("nuget", core.Manifest, &csprojParser{}, core.SuffixMatch(".fsproj")) core.Register("nuget", core.Manifest, &nuspecParser{}, core.SuffixMatch(".nuspec")) core.Register("nuget", core.Manifest, &packagesConfigParser{}, core.ExactMatch("packages.config")) + core.Register("nuget", core.Manifest, ¢ralPackagesParser{}, core.ExactMatch("Directory.Packages.props")) core.Register("nuget", core.Lockfile, &packagesLockParser{}, core.ExactMatch("packages.lock.json")) core.Register("nuget", core.Lockfile, &paketLockParser{}, core.ExactMatch("paket.lock")) core.Register("nuget", core.Lockfile, &projectAssetsParser{}, core.ExactMatch("project.assets.json")) @@ -44,14 +47,29 @@ type csprojPropertyGroup struct { } type csprojItemGroup struct { - PackageRefs []csprojPackageRef `xml:"PackageReference"` - References []csprojReference `xml:"Reference"` + Condition string `xml:"Condition,attr"` + PackageRefs []csprojPackageRef `xml:"PackageReference"` + PackageVersions []centralPackageVersion `xml:"PackageVersion"` + GlobalPackageRefs []centralPackageVersion `xml:"GlobalPackageReference"` + References []csprojReference `xml:"Reference"` } type csprojPackageRef struct { - Include string `xml:"Include,attr"` - Version string `xml:"Version,attr"` - VerElem string `xml:"Version"` + Include string `xml:"Include,attr"` + Update string `xml:"Update,attr"` + Condition string `xml:"Condition,attr"` + Version string `xml:"Version,attr"` + VerElem string `xml:"Version"` + VersionOverride string `xml:"VersionOverride,attr"` + VersionOverrideElem string `xml:"VersionOverride"` +} + +type centralPackageVersion struct { + Include string `xml:"Include,attr"` + Update string `xml:"Update,attr"` + Condition string `xml:"Condition,attr"` + Version string `xml:"Version,attr"` + VerElem string `xml:"Version"` } type csprojReference struct { @@ -59,6 +77,120 @@ type csprojReference struct { HintPath string `xml:"HintPath"` } +// appendNuGetDeclaration records a NuGet package reference at a +// case-insensitive logical location. +func appendNuGetDeclaration( + declarations *[]core.Declaration, + locations map[string]int, + prefix, name, version string, + scope core.Scope, +) { + name = strings.TrimSpace(name) + if name == "" { + return + } + location := core.NextLocation(locations, prefix+"/"+url.PathEscape(strings.ToLower(name))) + *declarations = append(*declarations, core.Declaration{ + Name: name, + Version: strings.TrimSpace(version), + Scope: scope, + Direct: true, + Location: location, + }) +} + +// packageReferenceName returns the Include or Update identity of a project +// package reference. +func packageReferenceName(ref csprojPackageRef) string { + if ref.Include != "" { + return ref.Include + } + return ref.Update +} + +// packageReferenceVersion returns the effective inline version declaration, +// preferring VersionOverride over Version. +func packageReferenceVersion(ref csprojPackageRef) string { + for _, version := range []string{ref.VersionOverride, ref.VersionOverrideElem, ref.Version, ref.VerElem} { + if version = strings.TrimSpace(version); version != "" { + return version + } + } + return "" +} + +// packageReferenceDependencyVersion returns the version syntax historically +// exposed through Dependencies for a project package reference. +func packageReferenceDependencyVersion(ref csprojPackageRef) string { + if ref.Version != "" { + return ref.Version + } + return strings.TrimSpace(ref.VerElem) +} + +// collectPackageReferences adds project PackageReference dependencies and +// declarations from one item group. +func collectPackageReferences( + group csprojItemGroup, + deps *[]core.Dependency, + declarations *[]core.Declaration, + seen map[string]bool, + locations map[string]int, +) { + prefix := "package-references" + if condition := strings.TrimSpace(group.Condition); condition != "" { + prefix += "/" + url.PathEscape(condition) + } + for _, ref := range group.PackageRefs { + name := packageReferenceName(ref) + if name == "" { + continue + } + version := packageReferenceVersion(ref) + refPrefix := prefix + if condition := strings.TrimSpace(ref.Condition); condition != "" { + refPrefix += "/" + url.PathEscape(condition) + } + appendNuGetDeclaration(declarations, locations, refPrefix, name, version, core.Runtime) + if ref.Include == "" { + continue + } + if seen[name] { + continue + } + seen[name] = true + *deps = append(*deps, core.Dependency{ + Name: name, + Version: packageReferenceDependencyVersion(ref), + Scope: core.Runtime, + Direct: true, + }) + } +} + +// collectAssemblyReferences adds legacy project Reference dependencies from +// one item group. +func collectAssemblyReferences(group csprojItemGroup, deps *[]core.Dependency, seen map[string]bool) { + for _, ref := range group.References { + if ref.Include == "" { + continue + } + + // Parse Include attribute: "Name, Version=x.x.x.x, Culture=neutral, ..." + name, version := parseReferenceInclude(ref.Include) + if name == "" || seen[name] || isSystemAssembly(name) { + continue + } + seen[name] = true + *deps = append(*deps, core.Dependency{ + Name: name, + Version: version, + Scope: core.Runtime, + Direct: true, + }) + } +} + func (p *csprojParser) Parse(filename string, content []byte) (*core.Result, error) { var project csprojProject if err := xml.Unmarshal(content, &project); err != nil { @@ -66,54 +198,13 @@ func (p *csprojParser) Parse(filename string, content []byte) (*core.Result, err } var deps []core.Dependency + var declarations []core.Declaration seen := make(map[string]bool) + locations := make(map[string]int) for _, group := range project.ItemGroups { - // Parse PackageReference elements - for _, ref := range group.PackageRefs { - name := ref.Include - if name == "" || seen[name] { - continue - } - seen[name] = true - - version := ref.Version - if version == "" { - version = strings.TrimSpace(ref.VerElem) - } - - deps = append(deps, core.Dependency{ - Name: name, - Version: version, - Scope: core.Runtime, - Direct: true, - }) - } - - // Parse Reference elements (legacy format) - for _, ref := range group.References { - if ref.Include == "" { - continue - } - - // Parse Include attribute: "Name, Version=x.x.x.x, Culture=neutral, ..." - name, version := parseReferenceInclude(ref.Include) - if name == "" || seen[name] { - continue - } - // Skip system assemblies - if isSystemAssembly(name) { - continue - } - seen[name] = true - - deps = append(deps, core.Dependency{ - Name: name, - Version: version, - Scope: core.Runtime, - Direct: true, - }) - } + collectPackageReferences(group, &deps, &declarations, seen, locations) + collectAssemblyReferences(group, &deps, seen) } // Default project name is the filename stem; PackageId or AssemblyName @@ -132,7 +223,73 @@ func (p *csprojParser) Parse(filename string, content []byte) (*core.Result, err } } - return &core.Result{Name: selfName, Version: selfVersion, Dependencies: deps}, nil + return &core.Result{ + Name: selfName, + Version: selfVersion, + Dependencies: deps, + Declarations: declarations, + }, nil +} + +// centralPackagesParser parses Directory.Packages.props files used by NuGet +// central package management. +type centralPackagesParser struct{} + +// collectCentralPackageItems records package versions or global package +// references from one central package item group. +func collectCentralPackageItems( + items []centralPackageVersion, + groupCondition, prefix string, + scope core.Scope, + deps *[]core.Dependency, + declarations *[]core.Declaration, + locations map[string]int, +) { + if condition := strings.TrimSpace(groupCondition); condition != "" { + prefix += "/" + url.PathEscape(condition) + } + for _, pkg := range items { + name := pkg.Include + if name == "" { + name = pkg.Update + } + version := pkg.Version + if version == "" { + version = strings.TrimSpace(pkg.VerElem) + } + if name == "" { + continue + } + *deps = append(*deps, core.Dependency{ + Name: name, + Version: version, + Scope: scope, + Direct: true, + }) + pkgPrefix := prefix + if condition := strings.TrimSpace(pkg.Condition); condition != "" { + pkgPrefix += "/" + url.PathEscape(condition) + } + appendNuGetDeclaration(declarations, locations, pkgPrefix, name, version, scope) + } +} + +func (p *centralPackagesParser) Parse(filename string, content []byte) (*core.Result, error) { + var project csprojProject + if err := xml.Unmarshal(content, &project); err != nil { + return nil, &core.ParseError{Filename: filename, Err: err} + } + + var deps []core.Dependency + var declarations []core.Declaration + locations := make(map[string]int) + for _, group := range project.ItemGroups { + collectCentralPackageItems(group.PackageVersions, group.Condition, "package-versions", core.Runtime, + &deps, &declarations, locations) + collectCentralPackageItems(group.GlobalPackageRefs, group.Condition, "global-package-references", core.Development, + &deps, &declarations, locations) + } + return &core.Result{Dependencies: deps, Declarations: declarations}, nil } // parseReferenceInclude parses a Reference Include attribute. @@ -208,11 +365,17 @@ func (p *nuspecParser) Parse(filename string, content []byte) (*core.Result, err } var deps []core.Dependency + var declarations []core.Declaration seen := make(map[string]bool) + locations := make(map[string]int) // Parse ungrouped dependencies for _, dep := range pkg.Metadata.Dependencies.Deps { - if dep.ID == "" || seen[dep.ID] { + if dep.ID == "" { + continue + } + appendNuGetDeclaration(&declarations, locations, "dependencies", dep.ID, dep.Version, core.Runtime) + if seen[dep.ID] { continue } seen[dep.ID] = true @@ -227,8 +390,16 @@ func (p *nuspecParser) Parse(filename string, content []byte) (*core.Result, err // Parse grouped dependencies for _, group := range pkg.Metadata.Dependencies.Groups { + prefix := "dependency-groups" + if framework := strings.TrimSpace(group.TargetFramework); framework != "" { + prefix += "/" + url.PathEscape(framework) + } for _, dep := range group.Deps { - if dep.ID == "" || seen[dep.ID] { + if dep.ID == "" { + continue + } + appendNuGetDeclaration(&declarations, locations, prefix, dep.ID, dep.Version, core.Runtime) + if seen[dep.ID] { continue } seen[dep.ID] = true @@ -246,6 +417,7 @@ func (p *nuspecParser) Parse(filename string, content []byte) (*core.Result, err Name: pkg.Metadata.ID, Version: pkg.Metadata.Version, Dependencies: deps, + Declarations: declarations, } licenseValue := strings.TrimSpace(pkg.Metadata.License.Value) switch strings.ToLower(strings.TrimSpace(pkg.Metadata.License.Type)) { @@ -279,6 +451,8 @@ func (p *packagesConfigParser) Parse(filename string, content []byte) (*core.Res } var deps []core.Dependency + var declarations []core.Declaration + locations := make(map[string]int) for _, pkg := range config.Packages { if pkg.ID == "" { @@ -296,9 +470,10 @@ func (p *packagesConfigParser) Parse(filename string, content []byte) (*core.Res Scope: scope, Direct: true, }) + appendNuGetDeclaration(&declarations, locations, "packages", pkg.ID, pkg.Version, scope) } - return &core.Result{Dependencies: deps}, nil + return &core.Result{Dependencies: deps, Declarations: declarations}, nil } // packagesLockParser parses packages.lock.json files. @@ -469,6 +644,8 @@ func (p *projectJSONParser) Parse(filename string, content []byte) (*core.Result } var deps []core.Dependency + var declarations []core.Declaration + locations := make(map[string]int) for name, value := range proj.Dependencies { version := "" @@ -487,9 +664,10 @@ func (p *projectJSONParser) Parse(filename string, content []byte) (*core.Result Scope: core.Runtime, Direct: true, }) + appendNuGetDeclaration(&declarations, locations, "dependencies", name, version, core.Runtime) } - return &core.Result{Dependencies: deps}, nil + return &core.Result{Dependencies: deps, Declarations: declarations}, nil } // libraryEntry holds the fields shared by deps.json and project.lock.json libraries. diff --git a/internal/nuget/nuget_test.go b/internal/nuget/nuget_test.go index dfa30c4..f8bd388 100644 --- a/internal/nuget/nuget_test.go +++ b/internal/nuget/nuget_test.go @@ -1,6 +1,7 @@ package nuget import ( + "net/url" "os" "testing" @@ -72,6 +73,111 @@ func TestCsproj(t *testing.T) { }) } +func TestCsprojDeclarations(t *testing.T) { + content := []byte(` + + + 2.0.0 + + + + + +`) + + result, err := (&csprojParser{}).Parse("example.csproj", content) + if err != nil { + t.Fatalf("Parse failed: %v", err) + } + condition := url.PathEscape("'$(TargetFramework)' == 'net8.0'") + itemCondition := url.PathEscape("'$(Configuration)' == 'Debug'") + want := map[string]string{ + "package-references/example": "1.0.0", + "package-references/example/2": "2.0.0", + "package-references/central": "3.0.0", + "package-references/" + condition + "/" + itemCondition + "/conditional": "4.0.0", + } + if len(result.Declarations) != len(want) { + t.Fatalf("Declarations has %d entries, want %d: %+v", len(result.Declarations), len(want), result.Declarations) + } + for _, declaration := range result.Declarations { + if version, ok := want[declaration.Location]; !ok || declaration.Version != version || !declaration.Direct { + t.Errorf("unexpected declaration: %+v", declaration) + } + } + if len(result.Dependencies) != 3 { + t.Fatalf("Dependencies has %d entries, want 3: %+v", len(result.Dependencies), result.Dependencies) + } + dependencyVersions := make(map[string]string) + for _, dependency := range result.Dependencies { + dependencyVersions[dependency.Name] = dependency.Version + } + if dependencyVersions["Example"] != "1.0.0" || + dependencyVersions["example"] != "" || + dependencyVersions["Conditional"] != "4.0.0" { + t.Errorf("Dependencies changed existing PackageReference versions: %+v", result.Dependencies) + } +} + +func TestCentralPackagesDeclarations(t *testing.T) { + content := []byte(` + + + 4.0.0 + + +`) + + result, err := (¢ralPackagesParser{}).Parse("Directory.Packages.props", content) + if err != nil { + t.Fatalf("Parse failed: %v", err) + } + want := map[string]string{ + "package-versions/newtonsoft.json": "13.0.3", + "package-versions/serilog": "4.0.0", + "package-versions/" + url.PathEscape("'$(TargetFramework)' == 'net8.0'") + "/conditional": "5.0.0", + } + if len(result.Declarations) != len(want) { + t.Fatalf("Declarations has %d entries, want %d: %+v", len(result.Declarations), len(want), result.Declarations) + } + for _, declaration := range result.Declarations { + if version, ok := want[declaration.Location]; !ok || declaration.Version != version || !declaration.Direct { + t.Errorf("unexpected declaration: %+v", declaration) + } + } +} + +func TestCentralPackagesGlobalReferences(t *testing.T) { + content := []byte(` + + + +`) + + result, err := (¢ralPackagesParser{}).Parse("Directory.Packages.props", content) + if err != nil { + t.Fatalf("Parse failed: %v", err) + } + if len(result.Declarations) != 1 { + t.Fatalf("Declarations has %d entries, want 1: %+v", len(result.Declarations), result.Declarations) + } + condition := url.PathEscape("'$(Configuration)' == 'Debug'") + declaration := result.Declarations[0] + if declaration.Location != "global-package-references/"+condition+"/nerdbank.gitversioning" || + declaration.Name != "Nerdbank.GitVersioning" || declaration.Version != "3.5.119" || + declaration.Scope != core.Development || !declaration.Direct { + t.Errorf("unexpected declaration: %+v", declaration) + } + if len(result.Dependencies) != 1 { + t.Fatalf("Dependencies has %d entries, want 1: %+v", len(result.Dependencies), result.Dependencies) + } + dependency := result.Dependencies[0] + if dependency.Name != "Nerdbank.GitVersioning" || dependency.Version != "3.5.119" || + dependency.Scope != core.Development || !dependency.Direct { + t.Errorf("unexpected dependency: %+v", dependency) + } +} + func TestNuspec(t *testing.T) { content, err := os.ReadFile("../../testdata/nuget/example.nuspec") if err != nil { @@ -116,6 +222,35 @@ func TestNuspec(t *testing.T) { // All dependencies are marked as Runtime } +func TestNuspecDeclarationsPreserveGroups(t *testing.T) { + content := []byte(` + + + +`) + + result, err := (&nuspecParser{}).Parse("example.nuspec", content) + if err != nil { + t.Fatalf("Parse failed: %v", err) + } + want := map[string]string{ + "dependencies/example": "[1.0.0]", + "dependency-groups/net8.0/example": "[2.0.0]", + "dependency-groups/any": "[3.0.0]", + } + if len(result.Declarations) != len(want) { + t.Fatalf("Declarations has %d entries, want %d: %+v", len(result.Declarations), len(want), result.Declarations) + } + for _, declaration := range result.Declarations { + if version, ok := want[declaration.Location]; !ok || declaration.Version != version { + t.Errorf("unexpected declaration: %+v", declaration) + } + } + if len(result.Dependencies) != 3 { + t.Fatalf("Dependencies has %d entries, want 3: %+v", len(result.Dependencies), result.Dependencies) + } +} + func TestPackagesConfig(t *testing.T) { content, err := os.ReadFile("../../testdata/nuget/packages.config") if err != nil { @@ -131,6 +266,9 @@ func TestPackagesConfig(t *testing.T) { if len(res.Dependencies) != 7 { t.Fatalf("expected 7 dependencies, got %d", len(res.Dependencies)) } + if len(res.Declarations) != 7 { + t.Fatalf("expected 7 declarations, got %d: %+v", len(res.Declarations), res.Declarations) + } depMap := make(map[string]core.Dependency) for _, d := range res.Dependencies { @@ -324,6 +462,9 @@ func TestProjectJSON(t *testing.T) { if len(res.Dependencies) != 13 { t.Fatalf("expected 13 dependencies, got %d", len(res.Dependencies)) } + if len(res.Declarations) != 13 { + t.Fatalf("expected 13 declarations, got %d: %+v", len(res.Declarations), res.Declarations) + } depMap := make(map[string]core.Dependency) for _, d := range res.Dependencies { diff --git a/internal/pypi/pypi.go b/internal/pypi/pypi.go index 177bb75..7bfd815 100644 --- a/internal/pypi/pypi.go +++ b/internal/pypi/pypi.go @@ -2,7 +2,6 @@ package pypi import ( "encoding/json" - "fmt" "maps" "net/url" "regexp" @@ -432,16 +431,12 @@ func appendPyPIDeclaration( return } identity := pypiNameSeparator.ReplaceAllString(strings.ToLower(name), "-") - base := prefix + "/" + url.PathEscape(identity) - locations[base]++ - location := base - if locations[base] > 1 { - location += fmt.Sprintf("/%d", locations[base]) - } + location := core.NextLocation(locations, prefix+"/"+url.PathEscape(identity)) *declarations = append(*declarations, core.Declaration{ Name: name, Version: strings.TrimSpace(version), Scope: scope, + Direct: true, Location: location, }) } diff --git a/manifests.go b/manifests.go index 44ac56c..c911e1d 100644 --- a/manifests.go +++ b/manifests.go @@ -119,7 +119,7 @@ func Parse(filename string, content []byte, opts ...Options) (*ParseResult, erro res.Dependencies[i].PURL = makePURL(eco, res.Dependencies[i].Name, version, res.Dependencies[i].RegistryURL) } for i := range res.Declarations { - res.Declarations[i].PURL = makePURL(eco, res.Declarations[i].Name, "", "") + res.Declarations[i].PURL = declarationPURL(eco, res.Declarations[i]) } return &ParseResult{ @@ -134,6 +134,15 @@ func Parse(filename string, content []byte, opts ...Options) (*ParseResult, erro }, nil } +// declarationPURL preserves a parser-supplied package identity or builds one +// from the parser's ecosystem. +func declarationPURL(ecosystem string, declaration core.Declaration) string { + if declaration.PURL != "" { + return declaration.PURL + } + return makePURL(ecosystem, declaration.Name, "", "") +} + // makePURL creates a Package URL for a dependency. func makePURL(ecosystem, name, version, registryURL string) string { return purl.BuildPURLString(ecosystem, name, version, registryURL) diff --git a/manifests_test.go b/manifests_test.go index a5bfdf9..7b59b4b 100644 --- a/manifests_test.go +++ b/manifests_test.go @@ -28,6 +28,7 @@ func TestParseAllEcosystems(t *testing.T) { {"golang go.sum", "testdata/golang/go.sum", "golang", Supplement}, {"pypi requirements.txt", "testdata/pypi/requirements.txt", "pypi", Manifest}, {"maven pom.xml", "testdata/maven/pom.xml", "maven", Manifest}, + {"nuget central packages", "testdata/nuget/Directory.Packages.props", "nuget", Manifest}, {"composer composer.json", "testdata/composer/composer.json", "composer", Manifest}, {"composer composer.lock", "testdata/composer/composer.lock", "composer", Lockfile}, } @@ -158,6 +159,42 @@ func TestDeclarationPURLs(t *testing.T) { wantVersion: "v4", wantPURL: "pkg:githubactions/actions/cache", }, + { + name: "cargo alias", + filename: "Cargo.toml", + content: "[dependencies]\nalias = { package = \"actual\", version = \"=1.0.0\" }\n", + location: "dependencies/alias", + wantName: "actual", + wantVersion: "=1.0.0", + wantPURL: "pkg:cargo/actual", + }, + { + name: "go requirement", + filename: "go.mod", + content: "module example.com/app\nrequire example.com/library v1.0.0\n", + location: "require/example.com%2Flibrary", + wantName: "example.com/library", + wantVersion: "v1.0.0", + wantPURL: "pkg:golang/example.com/library", + }, + { + name: "gleam dependency", + filename: "gleam.toml", + content: "[dependencies]\ngleam_stdlib = \"== 1.0.0\"\n", + location: "dependencies/gleam_stdlib", + wantName: "gleam_stdlib", + wantVersion: "== 1.0.0", + wantPURL: "pkg:hex/gleam_stdlib", + }, + { + name: "nuget central package", + filename: "Directory.Packages.props", + content: ``, + location: "package-versions/example", + wantName: "example", + wantVersion: "1.0.0", + wantPURL: "pkg:nuget/example", + }, } for _, test := range tests { @@ -171,7 +208,7 @@ func TestDeclarationPURLs(t *testing.T) { } declaration := result.Declarations[0] if declaration.Location != test.location || declaration.Name != test.wantName || - declaration.Version != test.wantVersion || declaration.PURL != test.wantPURL { + declaration.Version != test.wantVersion || declaration.PURL != test.wantPURL || !declaration.Direct { t.Errorf("Declaration = %+v, want location %q, name %q, version %q, PURL %q", declaration, test.location, test.wantName, test.wantVersion, test.wantPURL) } @@ -179,6 +216,14 @@ func TestDeclarationPURLs(t *testing.T) { } } +func TestDeclarationParserPURLIsPreserved(t *testing.T) { + declaration := Declaration{Name: "example", PURL: "pkg:npm/example"} + + if got, want := declarationPURL("deno", declaration), declaration.PURL; got != want { + t.Fatalf("declarationPURL() = %q, want %q", got, want) + } +} + func TestParseDeclaredLicenses(t *testing.T) { testCases := []struct { name string diff --git a/testdata/nuget/Directory.Packages.props b/testdata/nuget/Directory.Packages.props new file mode 100644 index 0000000..5d412a6 --- /dev/null +++ b/testdata/nuget/Directory.Packages.props @@ -0,0 +1,5 @@ + + + + +