Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -369,14 +369,26 @@ if !artifact.Observation.Complete {
log.Fatal("artifact body did not reach EOF")
}

sharedArtifact, err := artifact.Observation.Artifact(
"pkg:npm/lodash@4.17.21",
"lodash-4.17.21.tgz",
)
if err != nil {
log.Fatal(err)
}

fmt.Println(artifact.Observation.RequestedURL)
fmt.Println(artifact.Observation.FinalURL)
fmt.Println(artifact.Observation.ByteCount)
fmt.Println(artifact.Observation.Digests["sha256"])
fmt.Println(sharedArtifact.PURL)
fmt.Println(sharedArtifact.Digest)
```

The observation includes the time to receive the final response headers, status, declared size, media type, and an allow-list of response headers: `Accept-Ranges`, `Cache-Control`, `Content-Disposition`, `Content-Encoding`, `Content-Length`, `Content-Range`, `Digest`, `ETag`, `Expires`, and `Last-Modified`. SHA-256 and SHA-512 digests use lowercase hexadecimal encoding. Byte counts and digests remain unset until the stream reaches EOF, so a partial download cannot appear complete. Request and authentication headers are not copied into the observation.

After EOF, `FetchObservation.Artifact` converts the SHA-256 digest, byte count, and media type into an `artifacts.Artifact`. The caller supplies the package URL and filename. The conversion rejects incomplete observations and missing or malformed SHA-256 digests.

### Per-request headers

Use `FetchWithHeaders` to pass HTTP headers for a single request. This is useful when the auth token varies per request or is obtained dynamically (e.g. Docker Hub token exchange):
Expand Down
28 changes: 28 additions & 0 deletions fetch/observation.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,14 @@ import (
"crypto/sha256"
"crypto/sha512"
"encoding/hex"
"fmt"
"hash"
"io"
"net/http"
"time"

"github.com/git-pkgs/artifacts"
"github.com/opencontainers/go-digest"
)

var observedResponseHeaders = []string{
Expand Down Expand Up @@ -39,6 +43,30 @@ type FetchObservation struct {
Complete bool
}

// Artifact converts a completed observation into a shared artifact value.
func (observation *FetchObservation) Artifact(packageURL, filename string) (artifacts.Artifact, error) {
if observation == nil || !observation.Complete {
return artifacts.Artifact{}, fmt.Errorf("fetch observation: incomplete")
}

sha256Digest := observation.Digests["sha256"]
if sha256Digest == "" {
return artifacts.Artifact{}, fmt.Errorf("fetch observation: missing SHA-256 digest")
}

artifact, err := artifacts.New(
packageURL,
digest.Digest("sha256:"+sha256Digest),
observation.ByteCount,
filename,
observation.MediaType,
)
Comment thread
andrew marked this conversation as resolved.
if err != nil {
return artifacts.Artifact{}, fmt.Errorf("fetch observation: %w", err)
}
return artifact, nil
}

// ObservedArtifact contains an artifact and its fetch observation.
type ObservedArtifact struct {
*Artifact
Expand Down
90 changes: 90 additions & 0 deletions fetch/observation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
)

Expand Down Expand Up @@ -107,6 +108,29 @@ func TestFetchObserved(t *testing.T) {
if observation.Digests["sha512"] != hex.EncodeToString(sha512Sum[:]) {
t.Errorf("sha512 digest = %q", observation.Digests["sha512"])
}

sharedArtifact, err := observation.Artifact(
"pkg:npm/example@1.0.0?repository_url=https%3A%2F%2Fregistry.example.com&arch=arm64",
"artifact.tgz",
)
if err != nil {
t.Fatalf("Artifact failed: %v", err)
}
if sharedArtifact.PURL != "pkg:npm/example@1.0.0?arch=arm64&repository_url=https:%2F%2Fregistry.example.com" {
t.Errorf("PURL = %q", sharedArtifact.PURL)
}
Comment thread
andrew marked this conversation as resolved.
if sharedArtifact.Digest.Encoded() != observation.Digests["sha256"] {
t.Errorf("Digest = %q, want SHA-256 observation", sharedArtifact.Digest)
}
if sharedArtifact.Size != int64(len(content)) {
t.Errorf("Size = %d, want %d", sharedArtifact.Size, len(content))
}
if sharedArtifact.Filename != "artifact.tgz" {
t.Errorf("Filename = %q", sharedArtifact.Filename)
}
if sharedArtifact.MediaType != "application/gzip" {
t.Errorf("MediaType = %q", sharedArtifact.MediaType)
}
}

func TestFetchObservedIncompleteRead(t *testing.T) {
Expand Down Expand Up @@ -139,3 +163,69 @@ func TestFetchObservedIncompleteRead(t *testing.T) {
t.Errorf("Digests = %v after an incomplete read, want nil", artifact.Observation.Digests)
}
}

func TestFetchObservationArtifact(t *testing.T) {
validDigest := strings.Repeat("a", sha256.Size*2)
tests := []struct {
name string
observation *FetchObservation
wantSize int64
wantErr string
}{
{
name: "zero byte",
observation: &FetchObservation{
Complete: true,
Digests: map[string]string{"sha256": validDigest},
MediaType: "application/octet-stream",
},
wantSize: 0,
},
{
name: "incomplete",
observation: &FetchObservation{
Digests: map[string]string{"sha256": validDigest},
},
wantErr: "incomplete",
},
{
name: "nil",
observation: nil,
wantErr: "incomplete",
},
{
name: "missing digest",
observation: &FetchObservation{Complete: true},
wantErr: "missing SHA-256 digest",
},
{
name: "malformed digest",
observation: &FetchObservation{
Complete: true,
Digests: map[string]string{"sha256": "not-a-digest"},
},
wantErr: "digest",
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
artifact, err := test.observation.Artifact("pkg:pypi/example@1.0.0", "example.whl")
if test.wantErr != "" {
if err == nil {
t.Fatal("Artifact() error = nil")
}
if !strings.Contains(err.Error(), test.wantErr) {
t.Errorf("error = %q, want %q", err, test.wantErr)
}
return
}
if err != nil {
t.Fatalf("Artifact() error = %v", err)
}
if artifact.Size != test.wantSize {
t.Errorf("Size = %d, want %d", artifact.Size, test.wantSize)
}
})
}
}
6 changes: 4 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,18 @@ go 1.25.6

require (
github.com/cenk/backoff v2.2.1+incompatible
github.com/git-pkgs/artifacts v0.1.0
github.com/git-pkgs/pom v0.1.5
github.com/git-pkgs/purl v0.1.15
github.com/git-pkgs/purl v0.1.16
github.com/git-pkgs/spdx v0.3.0
github.com/opencontainers/go-digest v1.0.0
github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
github.com/rubyist/circuitbreaker v2.2.1+incompatible
)

require (
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
github.com/git-pkgs/vers v0.3.0 // indirect
github.com/git-pkgs/vers v0.3.1 // indirect
github.com/github/go-spdx/v2 v2.7.0 // indirect
github.com/package-url/packageurl-go v0.1.6 // indirect
github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea // indirect
Expand Down
12 changes: 8 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,20 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw=
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA=
github.com/git-pkgs/artifacts v0.1.0 h1:es73XxrsJefxO/DqtergTt+IaL+xPhzE0gSVrb6AFwk=
github.com/git-pkgs/artifacts v0.1.0/go.mod h1:zWZ0mrFi2M6ajGwpbo1nBGQTSb7qptP+zNG/EwAemBE=
github.com/git-pkgs/pom v0.1.5 h1:TGT8Az2OMxGWsXnSagtUMGzZm7Oax8HrSCteA+mi0qY=
github.com/git-pkgs/pom v0.1.5/go.mod h1:ufdMBe1lKzqOeP9IUb9NPZ458xKV8E8NvuyBMxOfwIk=
github.com/git-pkgs/purl v0.1.15 h1:iQ3clh0Cw41rkM0rf24B7ShnN9Z+UtLMAFlNDUs+Qd4=
github.com/git-pkgs/purl v0.1.15/go.mod h1:PqCLVBDeZrZgHysR803/AntMELgIr2LFZVNCcwLH2m0=
github.com/git-pkgs/purl v0.1.16 h1:VAX6tv0hhdTENbkrGMoPZbOAl1Y8U1/ZnzoCsYuNBYM=
github.com/git-pkgs/purl v0.1.16/go.mod h1:7u7ora8tQdrkS7Auclr5v8dCJdjN4ej6AbrvYZi2b7k=
github.com/git-pkgs/spdx v0.3.0 h1:AN0guJE7vN5gbOMi9We4j1ziS4cwgFVhTvjVDGfaC7Q=
github.com/git-pkgs/spdx v0.3.0/go.mod h1:cqRoZcvl530s/W+oGNvwjt4ODN8T1W6D/20MUZEFdto=
github.com/git-pkgs/vers v0.3.0 h1:xM4LLUCRmqzdDfe+/pVQUx4SRyFXRVth6tOsJ14wMKU=
github.com/git-pkgs/vers v0.3.0/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo=
github.com/git-pkgs/vers v0.3.1 h1:jy/ht2wIRJI5zQrccm6GTeYr+hGFwe2z8LV1HOr4Wco=
github.com/git-pkgs/vers v0.3.1/go.mod h1:biTbSQK1qdbrsxDEKnqe3Jzclxz8vW6uDcwKjfUGcOo=
github.com/github/go-spdx/v2 v2.7.0 h1:GzfXx4wFdlilARxmFRXW/mgUy3A4vSqZocCMFV6XFdQ=
github.com/github/go-spdx/v2 v2.7.0/go.mod h1:Ftc45YYG1WzpzwEPKRVm9Jv8vDqOrN4gWoCkK+bHer0=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/package-url/packageurl-go v0.1.6 h1:YO3p6u1XmCUliivUg/qWphaY8vI6hxSnnPv7Bfg3m5M=
github.com/package-url/packageurl-go v0.1.6/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0=
github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea h1:sKwxy1H95npauwu8vtF95vG/syrL0p8fSZo/XlDg5gk=
Expand Down