From c39ceac66f81b2495daf45e6bb425dcfe692107e Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Fri, 24 Jul 2026 11:58:43 +0100 Subject: [PATCH 01/21] add cloud driver Add the cloud driver with multi-region data-plane selection, optional provenance files, explicit cross-platform pulls, and upstream BuildKit integration. Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> Signed-off-by: Justin Chadwell Signed-off-by: Nick Adcock Signed-off-by: Nick Santos Signed-off-by: Sergio Garcez Signed-off-by: Silvin Lubecki --- build/build.go | 23 +- build/opt.go | 41 ++- builder/builder.go | 43 ++- cmd/buildx/main.go | 1 + docs/generate.go | 1 + docs/reference/buildx_create.md | 30 +- driver/cloud/create.go | 151 +++++++++++ driver/cloud/create_test.go | 102 +++++++ driver/cloud/driver.go | 468 ++++++++++++++++++++++++++++++++ driver/cloud/driver_test.go | 81 ++++++ driver/cloud/factory.go | 266 ++++++++++++++++++ driver/cloud/factory_test.go | 100 +++++++ driver/cloud/login.go | 241 ++++++++++++++++ driver/cloud/login_test.go | 57 ++++ 14 files changed, 1581 insertions(+), 24 deletions(-) create mode 100644 driver/cloud/create.go create mode 100644 driver/cloud/create_test.go create mode 100644 driver/cloud/driver.go create mode 100644 driver/cloud/driver_test.go create mode 100644 driver/cloud/factory.go create mode 100644 driver/cloud/factory_test.go create mode 100644 driver/cloud/login.go create mode 100644 driver/cloud/login_test.go diff --git a/build/build.go b/build/build.go index bef368ca6a4e..0c8253979cde 100644 --- a/build/build.go +++ b/build/build.go @@ -21,6 +21,7 @@ import ( noderesolver "github.com/docker/buildx/build/resolver" "github.com/docker/buildx/builder" "github.com/docker/buildx/driver" + "github.com/docker/buildx/driver/cloud" "github.com/docker/buildx/util/buildflags" "github.com/docker/buildx/util/confutil" "github.com/docker/buildx/util/desktop" @@ -97,6 +98,9 @@ type Options struct { GroupRef string Annotations map[exptypes.AnnotationKey]string // Not used during build, annotations are already set in Exports. Just used to check for support with drivers. Policy []buildflags.PolicyConfig + // onSolveResponse runs after BuildKit returns exporter metadata. Cloud pull + // uses it to import the image without a private BuildKit session exporter. + onSolveResponse func(context.Context, *client.SolveResponse) error } // ResourceLimits holds the cgroup resource constraints applied to individual @@ -231,7 +235,8 @@ type NamedContext struct { type reqForNode struct { *noderesolver.ResolvedNode - so *client.SolveOpt + so *client.SolveOpt + onSolveResponse func(context.Context, *client.SolveResponse) error } func filterAvailableNodes(nodes []builder.Node) ([]builder.Node, error) { @@ -281,6 +286,12 @@ func warnOnNoOutput(ctx context.Context, nodes []builder.Node, opts map[string]O var noOutputTargets []string for name, opt := range opts { if !opt.Linked && len(opt.Exports) == 0 { + // downstream toSolveOpt will force an image exporter if no exports are specified + // which, along with > 0 tags, will trigger a cloud pull in the cloud driver + implicitCloudPull := noMobyDriver.Factory().Name() == cloud.DriverName && len(opt.Tags) > 0 + if implicitCloudPull { + continue + } noOutputTargets = append(noOutputTargets, name) } } @@ -346,8 +357,9 @@ func newBuildRequests(ctx context.Context, docker *dockerutil.Client, cfg *confu } addGitAttrs(so) reqn = append(reqn, &reqForNode{ - ResolvedNode: np, - so: so, + ResolvedNode: np, + so: so, + onSolveResponse: localOpt.onSolveResponse, }) } reqForNodes[k] = reqn @@ -734,6 +746,11 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[ for k, v := range callRes { rr.ExporterResponse[k] = string(v) } + if onSolveResponse := reqForNodes[k][i].onSolveResponse; onSolveResponse != nil { + if err := onSolveResponse(ctx, rr); err != nil { + return err + } + } if opt.CallFunc == nil { rr.ExporterResponse["buildx.build.ref"] = buildRef if node.Driver.HistoryAPISupported(ctx) { diff --git a/build/opt.go b/build/opt.go index a4535e679ac1..cf9f03d693fd 100644 --- a/build/opt.go +++ b/build/opt.go @@ -23,6 +23,7 @@ import ( "github.com/distribution/reference" noderesolver "github.com/docker/buildx/build/resolver" "github.com/docker/buildx/driver" + "github.com/docker/buildx/driver/cloud" "github.com/docker/buildx/policy" "github.com/docker/buildx/util/buildflags" "github.com/docker/buildx/util/confutil" @@ -404,6 +405,12 @@ func toSolveOpt(ctx context.Context, np *noderesolver.ResolvedNode, multiDriver } } } + // modify exports to enable cloud pull. do this before setting up exporters, since + // the cloud pull might add or remove exporters. + if node.Driver.Factory().Name() == cloud.DriverName { + cloudDriver := node.Driver.Driver.(*cloud.Driver) + opt.Exports = cloudDriver.CheckCloudPull(ctx, docker, opt.Exports, opt.Tags) + } // fill in image exporter names from tags if len(opt.Tags) > 0 { @@ -417,7 +424,7 @@ func toSolveOpt(ctx context.Context, np *noderesolver.ResolvedNode, multiDriver } for i, e := range opt.Exports { switch e.Type { - case "image", "oci", "docker": + case "image", "oci", "docker", cloud.CloudPullExportType: opt.Exports[i].Attrs["name"] = strings.Join(tags, ",") } } @@ -509,6 +516,38 @@ func toSolveOpt(ctx context.Context, np *noderesolver.ResolvedNode, multiDriver opt.Exports[i].Attrs["buildinfo-attrs"] = v } } + if e.Type == cloud.CloudPullExportType { + // cloud pull if node is cloud driver + opt.Exports[i].Type = "image" + if node.Driver.Factory().Name() == cloud.DriverName { + if multiDriver { + return nil, nil, errors.Errorf("cloud pull for multi-node builds currently not supported") + } + if len(opt.Platforms) > 1 { + return nil, nil, errors.Errorf("cloud pull for single node multi-platform builds currently not supported") + } + cloudDriver := node.Driver.Driver.(*cloud.Driver) + targetPlatforms := slices.Clone(opt.Platforms) + tags := slices.Clone(opt.Tags) + opt.onSolveResponse = func(ctx context.Context, resp *client.SolveResponse) error { + pw := progress.ResetTime(pw) + return progress.Wrap("cloud pull", pw.Write, func(l progress.SubLogger) error { + imgDescriptor := resp.ExporterResponse[exptypes.ExporterImageDescriptorKey] + platform := resp.ExporterResponse[exptypes.ExporterPlatformsKey] + // Fallback: if platform is not in buildkit response + // but --platform was specified, use it + if platform == "" && len(targetPlatforms) == 1 { + platform = platforms.Format(targetPlatforms[0]) + } + + if err := cloudDriver.CloudPull(ctx, imgDescriptor, platform, docker, tags, l); err != nil { + return errors.Wrap(err, "pulling image from cloud") + } + return nil + }) + } + } + } if noDefaultOCIArtifact && supportAttestations { switch opt.Exports[i].Type { case client.ExporterImage, client.ExporterOCI, "moby": diff --git a/builder/builder.go b/builder/builder.go index fabd53f24507..4be402837173 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -12,6 +12,7 @@ import ( "time" "github.com/docker/buildx/driver" + "github.com/docker/buildx/driver/cloud" k8sutil "github.com/docker/buildx/driver/kubernetes/util" remoteutil "github.com/docker/buildx/driver/remote/util" "github.com/docker/buildx/localstate" @@ -360,9 +361,13 @@ func Create(ctx context.Context, txn *store.Txn, dockerCli command.Cli, opts Cre name := opts.Name if name == "" { - name, err = store.GenerateName(txn) - if err != nil { - return nil, err + if opts.Driver == cloud.DriverName && opts.Endpoint != "" { + name = "cloud-" + strings.ReplaceAll(strings.ToLower(opts.Endpoint), "/", "-") + } else { + name, err = store.GenerateName(txn) + if err != nil { + return nil, err + } } } @@ -492,6 +497,12 @@ func Create(ctx context.Context, txn *store.Txn, dockerCli command.Cli, opts Cre return nil, err } setEp = true + case driverName == cloud.DriverName: + if opts.Endpoint == "" { + return nil, errors.Errorf("no endpoint (builder) provided") + } + ep = cloud.EndpointPrefix + opts.Endpoint + setEp = false case opts.Endpoint != "": ep, err = validateEndpoint(dockerCli, opts.Endpoint) if err != nil { @@ -509,8 +520,30 @@ func Create(ctx context.Context, txn *store.Txn, dockerCli command.Cli, opts Cre setEp = false } - if err := ng.Update(opts.NodeName, ep, opts.Platforms, setEp, opts.Append, buildkitdFlags, buildkitdConfigFile, driverOpts); err != nil { - return nil, err + // support creating cloud builders by passing a builder group + // as the endpoint. This will create a node for each builder + // in the builder group. + // e.g. buildx create --driver cloud + if driverName == cloud.DriverName { + builders, err := cloud.GetBuilderInstances(ctx, opts.Endpoint, driverOpts) + if err != nil { + return nil, errors.Wrapf(err, "unknown builder") + } + for i, b := range builders { + if i != 0 { + opts.Append = true + } + + builderInstanceDriverOpts := cloud.GetBuilderInstanceDriverOpts(b, driverOpts) + if err := ng.Update(b.Name, b.Endpoint, []string{b.Platform}, false, opts.Append, buildkitdFlags, buildkitdConfigFile, builderInstanceDriverOpts); err != nil { + return nil, err + } + } + } else { + // default flow + if err := ng.Update(opts.NodeName, ep, opts.Platforms, setEp, opts.Append, buildkitdFlags, buildkitdConfigFile, driverOpts); err != nil { + return nil, err + } } if err := txn.Save(ng); err != nil { diff --git a/cmd/buildx/main.go b/cmd/buildx/main.go index 772566a845d4..24f68e1ae952 100644 --- a/cmd/buildx/main.go +++ b/cmd/buildx/main.go @@ -25,6 +25,7 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth/oidc" + _ "github.com/docker/buildx/driver/cloud" _ "github.com/docker/buildx/driver/docker" _ "github.com/docker/buildx/driver/docker-container" _ "github.com/docker/buildx/driver/kubernetes" diff --git a/docs/generate.go b/docs/generate.go index 9d5f7a92c5e5..e4975ab09874 100644 --- a/docs/generate.go +++ b/docs/generate.go @@ -19,6 +19,7 @@ import ( // import drivers otherwise factories are empty // for --driver output flag usage + _ "github.com/docker/buildx/driver/cloud" _ "github.com/docker/buildx/driver/docker" _ "github.com/docker/buildx/driver/docker-container" _ "github.com/docker/buildx/driver/kubernetes" diff --git a/docs/reference/buildx_create.md b/docs/reference/buildx_create.md index 666ecd95cf24..8032223fbe4b 100644 --- a/docs/reference/buildx_create.md +++ b/docs/reference/buildx_create.md @@ -9,21 +9,21 @@ Create a new builder instance ### Options -| Name | Type | Default | Description | -|:------------------------------------------|:--------------|:--------|:----------------------------------------------------------------------| -| [`--append`](#append) | `bool` | | Append a node to builder instead of changing it | -| `--bootstrap` | `bool` | | Boot builder after creation | -| [`--buildkitd-config`](#buildkitd-config) | `string` | | BuildKit daemon config file | -| [`--buildkitd-flags`](#buildkitd-flags) | `string` | | BuildKit daemon flags | -| `-D`, `--debug` | `bool` | | Enable debug logging | -| [`--driver`](#driver) | `string` | | Driver to use (available: `docker-container`, `kubernetes`, `remote`) | -| [`--driver-opt`](#driver-opt) | `stringArray` | | Options for the driver | -| [`--leave`](#leave) | `bool` | | Remove a node from builder instead of changing it | -| [`--name`](#name) | `string` | | Builder instance name | -| [`--node`](#node) | `string` | | Create/modify node with given name | -| [`--platform`](#platform) | `stringArray` | | Fixed platforms for current node | -| `--timeout` | `duration` | `20s` | Override the default timeout for loading builder status | -| [`--use`](#use) | `bool` | | Set the current builder instance | +| Name | Type | Default | Description | +|:------------------------------------------|:--------------|:--------|:-------------------------------------------------------------------------------| +| [`--append`](#append) | `bool` | | Append a node to builder instead of changing it | +| `--bootstrap` | `bool` | | Boot builder after creation | +| [`--buildkitd-config`](#buildkitd-config) | `string` | | BuildKit daemon config file | +| [`--buildkitd-flags`](#buildkitd-flags) | `string` | | BuildKit daemon flags | +| `-D`, `--debug` | `bool` | | Enable debug logging | +| [`--driver`](#driver) | `string` | | Driver to use (available: `cloud`, `docker-container`, `kubernetes`, `remote`) | +| [`--driver-opt`](#driver-opt) | `stringArray` | | Options for the driver | +| [`--leave`](#leave) | `bool` | | Remove a node from builder instead of changing it | +| [`--name`](#name) | `string` | | Builder instance name | +| [`--node`](#node) | `string` | | Create/modify node with given name | +| [`--platform`](#platform) | `stringArray` | | Fixed platforms for current node | +| `--timeout` | `duration` | `20s` | Override the default timeout for loading builder status | +| [`--use`](#use) | `bool` | | Set the current builder instance | diff --git a/driver/cloud/create.go b/driver/cloud/create.go new file mode 100644 index 000000000000..563e8d761948 --- /dev/null +++ b/driver/cloud/create.go @@ -0,0 +1,151 @@ +package cloud + +import ( + "context" + "encoding/json" + "fmt" + "maps" + "net/http" + "runtime" + "sort" + "strings" + "time" + + "github.com/pkg/errors" +) + +const hubHostDefault = "https://hub.docker.com" + +// CloudBuilder represents a builder instance in as returned by Hub Cloud Builds API. +type CloudBuilder struct { + // Name of the builder e.g. linux-amd64. + Name string `json:"name"` + // Platform of the builder e.g. linux/amd64. + Platform string `json:"platform"` + // Endpoint of the builder e.g. cloud://myorg/mybuilder_linux-amd64 + Endpoint string `json:"endpoint"` + // DataPlane contains data plane info for the builder + DataPlane CloudBuilderDataPlane `json:"data_plane"` +} + +type CloudBuilderDataPlane struct { + // Name is the logical name of the data plane e.g. us-east-1 + Name string `json:"name"` + // DisplayName is the name to display to users e.g. US East + DisplayName string `json:"display_name"` + // ProxyEndpoint is the address of the data plane's proxy API e.g. tcp://build-cloud.docker.com:443 + ProxyEndpoint string `json:"proxy_endpoint"` + // RegistryEndpoint is the address of the data plane's registry API e.g. https://build-cloud.docker.com:443 + RegistryEndpoint string `json:"registry_endpoint"` + // HealthEndpoint is the address of the data plane's health check API e.g. https://build-cloud.docker.com:443/v2 + HealthEndpoint string `json:"health_endpoint"` +} + +// Arch returns the architecture of the builder. +// example: for platform linux/amd64, returns amd64. +func (b CloudBuilder) Arch() string { + parts := strings.Split(b.Platform, "/") + if len(parts) != 2 { + return "" + } + return parts[1] +} + +// GetBuilderInstances calls the Hub Cloud Builds API to get +// a list of builder instances under a builder group. +func GetBuilderInstances(ctx context.Context, group string, driverOpts map[string]string) ([]CloudBuilder, error) { + ctx, cancel := context.WithTimeoutCause(ctx, 30*time.Second, errors.WithStack(context.DeadlineExceeded)) + defer cancel() + registryHostname := hubRegistryEntry + hubHost := hubHostDefault + for k, v := range driverOpts { + switch k { + case "internal.auth.entry": + if err := verifyDomain(v); err != nil { + return nil, err + } + registryHostname = v + case "internal.hub.host": + hubHost = v + } + } + + token, err := fetchHubToken(ctx, registryHostname, hubHost) + if err != nil { + return nil, errors.Wrap(err, "failed to fetch hub token") + } + + builders, err := getBuilderInstances(ctx, group, hubHost, token) + if err != nil { + return nil, errors.Wrap(err, "failed to get builders") + } + + if len(builders) == 0 { + return nil, errors.Errorf("no builders found for group: %s", group) + } + + // Bubble builders with arch matching local to the front. + // Preserves order otherwise. + sort.SliceStable(builders, func(i, j int) bool { + return builders[i].Arch() == runtime.GOARCH && builders[j].Arch() != runtime.GOARCH + }) + + return builders, nil +} + +// GetBuilderInstanceDriverOpts creates the instance specific driver opts +// from the driver-opt args and the builder instance details +func GetBuilderInstanceDriverOpts(builderInstance CloudBuilder, driverOpts map[string]string) map[string]string { + opts := maps.Clone(driverOpts) + // If builder instance has a data plane endpoint and it has not been overridden by the driver opts then apply it + if opts[optKeyInternalAddress] == "" { + if opts == nil { + opts = make(map[string]string) + } + if opts[optKeyInternalProxyAddress] == "" && builderInstance.DataPlane.ProxyEndpoint != "" { + opts[optKeyInternalProxyAddress] = builderInstance.DataPlane.ProxyEndpoint + } + if opts[optKeyInternalRegistryAddress] == "" && builderInstance.DataPlane.RegistryEndpoint != "" { + opts[optKeyInternalRegistryAddress] = builderInstance.DataPlane.RegistryEndpoint + } + if opts[optKeyInternalHealthAddress] == "" && builderInstance.DataPlane.HealthEndpoint != "" { + opts[optKeyInternalHealthAddress] = builderInstance.DataPlane.HealthEndpoint + } + } + return opts +} + +// getBuilderInstances calls the Hub Cloud Builds API to get +// a list of builder instances under a builder group. +func getBuilderInstances(ctx context.Context, builderGroup, hubHost, token string) ([]CloudBuilder, error) { + parts := strings.Split(builderGroup, "/") + namespace := parts[0] + group := parts[1] + url := fmt.Sprintf("%s/v2/cloud-builds/accounts/%s/builder-groups/%s/instances", hubHost, namespace, group) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, errors.Wrap(err, "create request") + } + req.Header.Set("Authorization", "Bearer "+token) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, errors.Wrap(err, "request failed") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, errors.New("response status code: " + resp.Status) + } + + // ignoring pagination as the default page size of 10 should include all builders. + var respPayload struct { + Results []CloudBuilder `json:"results"` + } + if err := json.NewDecoder(resp.Body).Decode(&respPayload); err != nil { + return nil, errors.Wrap(err, "decode get instances response") + } + + return respPayload.Results, nil +} diff --git a/driver/cloud/create_test.go b/driver/cloud/create_test.go new file mode 100644 index 000000000000..ecc958ad05a9 --- /dev/null +++ b/driver/cloud/create_test.go @@ -0,0 +1,102 @@ +package cloud + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetBuilderInstanceDriverOpts(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + driverOpts map[string]string + builderInstance CloudBuilder + expected map[string]string + }{ + { + name: "NoOptsOrDataplane", + driverOpts: map[string]string{}, + expected: map[string]string{}, + }, + { + name: "DataPlaneOnly", + builderInstance: CloudBuilder{ + DataPlane: CloudBuilderDataPlane{ + ProxyEndpoint: "tcp://endpoint:443", + RegistryEndpoint: "https://endpoint:443", + HealthEndpoint: "https://endpoint:443/v2/", + }, + }, + driverOpts: map[string]string{ + "a": "b", + }, + expected: map[string]string{ + "a": "b", + "internal.cloud.proxy.address": "tcp://endpoint:443", + "internal.cloud.registry.address": "https://endpoint:443", + "internal.cloud.health.address": "https://endpoint:443/v2/", + }, + }, + { + name: "DriverOptsOnly", + driverOpts: map[string]string{ + "a": "b", + "internal.address": "c", + }, + expected: map[string]string{ + "a": "b", + "internal.address": "c", + }, + }, + { + name: "InternalAddressOverridesDataPlane", + builderInstance: CloudBuilder{ + DataPlane: CloudBuilderDataPlane{ + ProxyEndpoint: "tcp://endpoint:443", + RegistryEndpoint: "https://endpoint:443", + HealthEndpoint: "https://endpoint:443/v2/", + }, + }, + driverOpts: map[string]string{ + "a": "b", + "internal.address": "c", + }, + expected: map[string]string{ + "a": "b", + "internal.address": "c", + }, + }, + { + name: "PartialDataPlaneOverride", + builderInstance: CloudBuilder{ + DataPlane: CloudBuilderDataPlane{ + ProxyEndpoint: "tcp://endpoint:443", + RegistryEndpoint: "https://endpoint:443", + HealthEndpoint: "https://endpoint:443/v2/", + }, + }, + driverOpts: map[string]string{ + "a": "b", + "internal.cloud.proxy.address": "tcp://endpoint2:443", + }, + expected: map[string]string{ + "a": "b", + "internal.cloud.proxy.address": "tcp://endpoint2:443", + "internal.cloud.registry.address": "https://endpoint:443", + "internal.cloud.health.address": "https://endpoint:443/v2/", + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + result := GetBuilderInstanceDriverOpts(tc.builderInstance, tc.driverOpts) + + assert.Equal(t, tc.expected, result) + }) + } +} diff --git a/driver/cloud/driver.go b/driver/cloud/driver.go new file mode 100644 index 000000000000..0ae0443dee7b --- /dev/null +++ b/driver/cloud/driver.go @@ -0,0 +1,468 @@ +package cloud + +import ( + "compress/gzip" + "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "maps" + "net" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/containerd/platforms" + "github.com/docker/buildx/driver" + "github.com/docker/buildx/util/dockerutil" + "github.com/docker/buildx/util/progress" + "github.com/moby/moby/api/types/jsonstream" + "github.com/moby/moby/api/types/registry" + "github.com/moby/moby/api/types/system" + dockerclient "github.com/moby/moby/client" + "github.com/sirupsen/logrus" + + "github.com/Masterminds/semver/v3" + "github.com/moby/buildkit/client" + specs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "google.golang.org/grpc" + grpcgzip "google.golang.org/grpc/encoding/gzip" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/metadata" +) + +func init() { + if err := grpcgzip.SetLevel(gzip.BestCompression); err != nil { + panic(err) + } +} + +type Driver struct { + factory driver.Factory + driver.InitConfig + + defaultLoad bool + builder string + tokenSource tokenSource + proxyAddress string + registryAddress string + healthAddress string + tls tlsOpts + + headers []string +} + +// make sure cloud driver implements Driver interface +var _ driver.Driver = &Driver{} + +// tlsOpts allows overriding the default TLS configuration for communicating +// with the registry. This should only be used for development and testing. +type tlsOpts struct { + insecure bool + serverName string + caCert string +} + +func (d *Driver) Dial(ctx context.Context) (net.Conn, error) { + return nil, errors.New("cloud driver does not support dialing") +} + +func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error { + return nil +} + +func (d *Driver) Info(ctx context.Context) (*driver.Info, error) { + // hit the /v2 to check if the registry is up + url := d.healthAddress + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + tlsConfig, err := d.tlsConfig() + if err != nil { + return nil, errors.Wrapf(err, "failed to create TLS config: %v", err) + } + client := newClient(&http.Transport{ + TLSClientConfig: tlsConfig, + }) + + resp, err := client.Do(req) + if resp != nil { + defer drainResponse(resp) + } + if err != nil || resp.StatusCode != http.StatusUnauthorized { + return &driver.Info{ + Status: driver.Inactive, + }, nil + } + + return &driver.Info{ + Status: driver.Running, + }, nil +} + +func (d *Driver) Version(ctx context.Context) (string, error) { + return "", nil +} + +func (d *Driver) Stop(ctx context.Context, force bool) error { + return nil +} + +func (d *Driver) Rm(ctx context.Context, force, rmVolume, rmDaemon bool) error { + return nil +} + +func (d *Driver) Client(ctx context.Context, opts ...client.ClientOpt) (*client.Client, error) { + if !d.tls.insecure { + if d.tls.caCert != "" { + opts = append(opts, client.WithServerConfig(d.tls.serverName, d.tls.caCert)) + } else { + opts = append(opts, client.WithServerConfigSystem(d.tls.serverName)) + } + } + + kp := keepalive.ClientParameters{Time: 10 * time.Second} + opts = append(opts, client.WithGRPCDialOption(grpc.WithKeepaliveParams(kp))) + + i := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + token, err := d.tokenSource(ctx) + if err != nil { + return errors.Wrap(err, "fetching auth token") + } + headers := append(d.headers, authorizationKey, "Bearer "+token) + ctx = metadata.AppendToOutgoingContext(ctx, headers...) + opts = append([]grpc.CallOption{grpc.UseCompressor(grpcgzip.Name)}, opts...) + return invoker(ctx, method, req, reply, cc, opts...) + } + i2 := func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + token, err := d.tokenSource(ctx) + if err != nil { + return nil, errors.Wrap(err, "fetching auth token") + } + headers := append(d.headers, authorizationKey, "Bearer "+token) + ctx = metadata.AppendToOutgoingContext(ctx, headers...) + opts = append([]grpc.CallOption{grpc.UseCompressor(grpcgzip.Name)}, opts...) + return streamer(ctx, desc, cc, method, opts...) + } + opts = append(opts, + []client.ClientOpt{ + client.WithGRPCDialOption(grpc.WithChainUnaryInterceptor(i)), + client.WithGRPCDialOption(grpc.WithChainStreamInterceptor(i2)), + }..., + ) + c, err := client.New(ctx, d.proxyAddress, opts...) + if err != nil { + return nil, err + } + return c, nil +} + +func (d *Driver) Features(_ context.Context) map[driver.Feature]bool { + return map[driver.Feature]bool{ + driver.OCIExporter: true, + driver.DockerExporter: false, + driver.CacheExport: true, + driver.MultiPlatform: true, + driver.DirectPush: true, + driver.DefaultLoad: d.defaultLoad, + } +} + +func (d *Driver) Factory() driver.Factory { + return d.factory +} + +func (d *Driver) IsMobyDriver() bool { + return false +} + +func (d *Driver) Config() driver.InitConfig { + return d.InitConfig +} + +func (d *Driver) HostGatewayIP(ctx context.Context) (net.IP, error) { + return nil, errors.New("host-gateway is not supported by the cloud driver") +} + +// CloudPull pulls an image from the cloud registry. +// It should be called right after Solve returns. +func (d *Driver) CloudPull(ctx context.Context, imageDescriptor string, platform string, duc *dockerutil.Client, tags []string, l progress.SubLogger) error { + dc, err := duc.API("") + if err != nil { + return err + } + + token, err := d.tokenSource(ctx) + if err != nil { + return errors.Wrap(err, "fetching auth token") + } + + hydroRegistryAddr := d.registryAddress + + // Reuse the same token used for the build + authConfig := registry.AuthConfig{ + RegistryToken: token, + ServerAddress: hydroRegistryAddr, + } + authJSON, err := json.Marshal(authConfig) + if err != nil { + return err + } + base64Auth := base64.URLEncoding.EncodeToString(authJSON) + var pullPlatforms []specs.Platform + if platform != "" { + targetPlatform, err := platforms.Parse(platform) + if err != nil { + return errors.Wrapf(err, "parsing platform %s", platform) + } + pullPlatforms = []specs.Platform{platforms.Normalize(targetPlatform)} + } + pullOpts := dockerclient.ImagePullOptions{ + RegistryAuth: base64Auth, + Platforms: pullPlatforms, + } + + // Find manifest to pull + decodedDescriptor, err := base64.StdEncoding.DecodeString(imageDescriptor) + if err != nil { + return err + } + var desc specs.Descriptor + if err := json.Unmarshal(decodedDescriptor, &desc); err != nil { + return errors.Wrapf(err, "unmarshal descriptor %s", decodedDescriptor) + } + sha := desc.Digest.String() + + // Use builder name as tag (will be replaced with user tags after the pull) + ref := fmt.Sprintf("%s/%s@%s", hydroRegistryAddr, d.builder, sha) + + pullResp, err := dc.ImagePull(ctx, ref, pullOpts) + if err != nil { + return err + } + defer pullResp.Close() + + if err := printMagicPull(pullResp, l); err != nil { + return err + } + + // Tag image with user tags and remove the builder name tag + for _, tag := range tags { + opts := dockerclient.ImageTagOptions{ + Source: ref, + Target: tag, + } + if _, err = dc.ImageTag(ctx, opts); err != nil { + return err + } + } + + // TODO: support case where no tags are provided + removeOpts := dockerclient.ImageRemoveOptions{PruneChildren: false} + _, err = dc.ImageRemove(ctx, ref, removeOpts) + if err != nil { + logrus.Debugf("failed to remove temp tag %s: %v", ref, err) + } + + return nil +} + +// CheckCloudPull checks if cloud pull should be enabled and returns the updated exports. +// Enables cloud pull if +// - cloud driver is used +// - daemon supports RegistryToken in ImagePullOptions +// - docker exporter or image exporter (without push) is used +// - at least one tag is set +func (d *Driver) CheckCloudPull(ctx context.Context, docker *dockerutil.Client, exports []client.ExportEntry, tags []string) []client.ExportEntry { + if len(tags) == 0 { + return exports + } + + supportsRegistryTokenPull, err := daemonSupportsRegistryTokenPull(ctx, docker) + if err != nil { + logrus.Warnf("failed to detect registry pull token support in daemon: %v", err) + return exports + } + + if !supportsRegistryTokenPull { + return exports + } + + if len(exports) == 0 { + attrs := maps.Clone(CouldPullExportRequiredAttributes) + return []client.ExportEntry{ + { + Type: CloudPullExportType, + Attrs: attrs, + }, + } + } + + for i, e := range exports { + pushAttr, _ := strconv.ParseBool(e.Attrs["push"]) + if e.Type == "docker" || (e.Type == "image" && !pushAttr) { + exports[i].Type = CloudPullExportType + maps.Copy(exports[i].Attrs, CouldPullExportRequiredAttributes) + } + } + + return exports +} + +// tlsConfig returns a TLS configuration for communicating with the registry, +// based on d.tls. Note that this is only intended for development and testing. +func (d *Driver) tlsConfig() (*tls.Config, error) { + tlsConfig := &tls.Config{ + InsecureSkipVerify: d.tls.insecure, + } + if d.tls.caCert != "" { + // Prefer the system cert pool if available, but fallback + // to an empty pool + rootCAs, _ := x509.SystemCertPool() + if rootCAs == nil { + rootCAs = x509.NewCertPool() + } + ca, err := os.ReadFile(d.tls.caCert) + if err != nil { + return nil, errors.Wrapf(err, "could not read CA certificate %s: %v", d.tls.caCert, err) + } + if ok := rootCAs.AppendCertsFromPEM(ca); !ok { + return nil, errors.Errorf("failed to add CA certificate %s to root CAs", d.tls.caCert) + } + tlsConfig.RootCAs = rootCAs + } + return tlsConfig, nil +} + +func printMagicPull(rc io.Reader, l progress.SubLogger) (err error) { + started := map[string]client.VertexStatus{} + + defer func() { + if err != nil { + return + } + for _, st := range started { + if st.Completed == nil { + now := time.Now() + st.Completed = &now + l.SetStatus(&st) + } + } + }() + + dec := json.NewDecoder(rc) + + var ( + parsedError error + jm jsonstream.Message + ) + + for { + if err = dec.Decode(&jm); err != nil { + if parsedError != nil { + err = parsedError + } else if err == io.EOF { + err = nil + } + return + } + + if jm.Error != nil { + parsedError = jm.Error + } + + if jm.ID == "" { + continue + } + + // handle temporary fake registry tags + if strings.ContainsRune(jm.ID, '@') { + continue + } + if strings.ContainsRune(jm.Status, ':') { + continue + } + + id := "pulling layer " + jm.ID + st, ok := started[id] + if !ok { + if jm.Progress != nil || strings.HasPrefix(jm.Status, "Pulling") || strings.HasPrefix(jm.Status, "Already exists") { + now := time.Now() + st = client.VertexStatus{ + ID: id, + Started: &now, + } + } else { + continue + } + } + st.Timestamp = time.Now() + if jm.Progress != nil && jm.Status == "Downloading" { + st.Current = jm.Progress.Current + st.Total = jm.Progress.Total + } + if jm.Error != nil { + now := time.Now() + st.Completed = &now + } + + if jm.Status == "Pull complete" || jm.Status == "Already exists" { + now := time.Now() + st.Completed = &now + st.Current = st.Total + } + started[id] = st + l.SetStatus(&st) + } +} + +func daemonSupportsRegistryTokenPull(ctx context.Context, duc *dockerutil.Client) (bool, error) { + dc, err := duc.API("") + if err != nil { + return false, err + } + + resp, err := dc.Info(ctx, dockerclient.InfoOptions{}) + if err != nil { + return false, err + } + return daemonInfoSupportsRegistryTokenPull(resp.Info) +} + +func daemonInfoSupportsRegistryTokenPull(resp system.Info) (bool, error) { + for _, status := range resp.DriverStatus { + if len(status) == 2 && status[1] == "io.containerd.snapshotter.v1" { + // If we're using the containerd snapshotter, we need + // to check for https://github.com/moby/moby/pull/46475. + // + // Simplest way: + // - if we're using docker desktop, assume we have a compatible version + // - otherwise, check if the daemon is on 25.0.0 or newer + constraint, err := semver.NewConstraint(">= 25.0.0") + if err != nil { + return false, err + } + + if resp.OperatingSystem == "Docker Desktop" { + return true, nil + } + + version, err := semver.NewVersion(resp.ServerVersion) + if err != nil { + return false, err + } + + return constraint.Check(version), nil + } + } + + return true, nil +} diff --git a/driver/cloud/driver_test.go b/driver/cloud/driver_test.go new file mode 100644 index 000000000000..70a52f1e9ac9 --- /dev/null +++ b/driver/cloud/driver_test.go @@ -0,0 +1,81 @@ +package cloud + +import ( + "fmt" + "strings" + "testing" + + "github.com/moby/moby/api/types/system" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type registryTokenSupport struct { + os string + version string + driver string + expected bool + errCheck func(t *testing.T, err error) +} + +func TestRegistryTokenSupport(t *testing.T) { + t.Parallel() + + table := []registryTokenSupport{ + { + os: "Docker Desktop", + version: "master", + driver: "io.containerd.snapshotter.v1", + expected: true, + }, + { + os: "Linux Mint 21.1", + version: "25.0.0", + driver: "io.containerd.snapshotter.v1", + expected: true, + }, + { + os: "Linux Mint 21.1", + version: "24.0.6", + driver: "io.containerd.snapshotter.v1", + expected: false, + }, + { + os: "Linux Mint 21.1", + version: "master", + driver: "io.containerd.snapshotter.v1", + errCheck: func(t *testing.T, err error) { + require.Error(t, err) + // Some versions will return a capitalized error, so we use toLower to normalize the error message + assert.Contains(t, strings.ToLower(err.Error()), "invalid semantic version") + }, + expected: false, + }, + { + os: "Linux Mint 21.1", + version: "24.0.6", + driver: "docker", + expected: true, + }, + } + + for i, c := range table { + t.Run(fmt.Sprintf("case%d", i), func(t *testing.T) { + t.Parallel() + + actual, err := daemonInfoSupportsRegistryTokenPull(system.Info{ + ServerVersion: c.version, + OperatingSystem: c.os, + DriverStatus: [][2]string{ + {"driver-type", c.driver}, + }, + }) + if c.errCheck != nil { + c.errCheck(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, c.expected, actual) + }) + } +} diff --git a/driver/cloud/factory.go b/driver/cloud/factory.go new file mode 100644 index 000000000000..32f18a62e1d5 --- /dev/null +++ b/driver/cloud/factory.go @@ -0,0 +1,266 @@ +package cloud + +import ( + "context" + "net/url" + "os" + "runtime" + "runtime/debug" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/docker/buildx/driver" + "github.com/docker/buildx/version" + dockerclient "github.com/moby/moby/client" + "github.com/pkg/errors" +) + +const EndpointPrefix = "cloud://" + +const ( + DriverName = "cloud" + // CloudPullExportType is the type of export used for enabling + // cloud pulls. + CloudPullExportType = "cloud" + AuthHost = "https://auth.docker.io" + ProxyAddress = "tcp://build-cloud.docker.com:443" + RegistryAddress = "build-cloud.docker.com:443" + HealthAddress = "https://build-cloud.docker.com:443/v2/" + + // We want a low priority for the cloud driver for now. + lowPriority = 1000 + hubRegistryEntry = "https://index.docker.io/v1/" + service = "service=build.docker.com" + // authorizationKey is the header for bearer token + authorizationKey = "authorization" + // buildxVersionKey is the header to identify the client version, eg: v0.5.1-desktop.1 + buildxVersionKey = "buildx-version" + // userContextKey is the header to specify where the build is occurring, eg: local, ci, github, gitlab, circleci, travis, buildkite, jenkins + userContextKey = "user-context" + // userPlatformKey is the header to specify the platform which triggered the build, eg: linux/amd64, darwin/amd64 + userPlatformKey = "user-platform" + // optKeyInternalAddress key to store the optional address for cloud endpoints + // overrides specific internal.cloud.* addresses + optKeyInternalAddress = "internal.address" + // optKeyInternalAddress key to store the optional buildkit proxy address + optKeyInternalProxyAddress = "internal.cloud.proxy.address" + // optKeyInternalRegistryAddress key to store the optional cloud registry address for cloud-pull + // defaults to `internal.address` on https + optKeyInternalRegistryAddress = "internal.cloud.registry.address" + // optKeyInternalHealthAddress key to store the optional cloud health check address + // defaults to `internal.cloud.registry.address`/v2 + optKeyInternalHealthAddress = "internal.cloud.health.address" +) + +var ( + // CouldPullExportRequiredAttributes is the list of required attributes for the Export entry + // when doing a cloud pull. + CouldPullExportRequiredAttributes = map[string]string{ + "attestation-inline": "false", + } +) + +func init() { + driver.Register(&factory{}) +} + +type factory struct { + headers []string + once sync.Once +} + +func (*factory) Name() string { + return DriverName +} + +func (*factory) Usage() string { + return DriverName +} + +func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient, _ map[string][]string) int { + return lowPriority +} + +func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver, error) { + if len(cfg.Files) > 0 { + return nil, errors.Errorf("setting config file is not supported for " + DriverName + " driver") + } + if len(cfg.BuildkitdFlags) > 0 { + return nil, errors.Errorf("setting buildkit flags is not supported for " + DriverName + " driver") + } + f.prepareHeaders() + d := &Driver{ + factory: f, + InitConfig: cfg, + proxyAddress: ProxyAddress, + registryAddress: RegistryAddress, + healthAddress: HealthAddress, + builder: strings.TrimPrefix(cfg.EndpointAddr, EndpointPrefix), + headers: f.headers, + } + + authHost := AuthHost + authEntry := hubRegistryEntry + for k, v := range cfg.DriverOpts { + switch k { + case "default-load": + parsed, err := strconv.ParseBool(v) + if err != nil { + return nil, err + } + d.defaultLoad = parsed + case "builder", "org": + // kept for backwards compat + d.builder = v + case optKeyInternalAddress: + // Full override of cloud endpoints + if err := verifyDomain(v); err != nil { + return nil, err + } + d.proxyAddress = v + // registry should not have a protocol, https is implicit + d.registryAddress = strings.TrimPrefix(v, "tcp://") + registryHTTPAddress := strings.Replace(v, "tcp", "https", 1) + d.healthAddress = registryHTTPAddress + "/v2/" + case optKeyInternalProxyAddress: + if err := verifyDomain(v); err != nil { + return nil, err + } + d.proxyAddress = v + case optKeyInternalRegistryAddress: + if err := verifyDomain(v); err != nil { + return nil, err + } + d.registryAddress = v + case optKeyInternalHealthAddress: + if err := verifyDomain(v); err != nil { + return nil, err + } + d.healthAddress = v + case "internal.auth.host": + if err := verifyDomain(v); err != nil { + return nil, err + } + authHost = v + case "internal.auth.entry": + if err := verifyDomain(v); err != nil { + return nil, err + } + authEntry = v + case "internal.tls.insecure": + b, err := strconv.ParseBool(v) + if err != nil { + return nil, errors.Wrapf(err, "invalid value for %s", k) + } + d.tls.insecure = b + case "internal.tls.servername": + d.tls.serverName = v + case "internal.tls.cacert": + d.tls.caCert = v + case "internal.hub.host": + // used to call Cloud Builds API in group creation flow. + default: + return nil, errors.Errorf("invalid driver option %s for cloud driver", k) + } + } + + if d.builder == "" { + return nil, errors.Errorf("builder name missing") + } + + if len(strings.Split(d.builder, "/")) != 2 { + return nil, errors.Errorf("builder should be in the format: /") + } + + d.tokenSource = newTokenSource(func(ctx context.Context) (string, error) { + return login(ctx, authEntry, authHost, d.builder) + }, time.Now) + + return d, nil +} + +func (f *factory) prepareHeaders() { + f.once.Do(func() { + f.headers = []string{ + buildxVersionKey, getBuildxVersion(), + userContextKey, getUserContext(os.Environ()), + userPlatformKey, runtime.GOOS + "/" + runtime.GOARCH, + } + }) +} + +func (f *factory) AllowsInstances() bool { + return true +} + +// When called from desktop build backend, the version.Version is unknown. We try to get the version from the build info. +func getBuildxVersion() string { + if version.Version != "v0.0.0+unknown" { + return version.Version + } + info, ok := debug.ReadBuildInfo() + if !ok { + return version.Version + } + for _, dep := range info.Deps { + if dep.Path == "github.com/docker/buildx" { + if dep.Replace != nil { + return dep.Replace.Version + } + return dep.Version + } + } + + return version.Version +} + +func getUserContext(env []string) string { + if !slices.Contains(env, "CI=true") { + return "local" + } + switch { + case slices.Contains(env, "GITHUB_ACTIONS=true"): + return "github" + case slices.Contains(env, "GITLAB_CI=true"): + return "gitlab" + case slices.Contains(env, "CIRCLECI=true"): + return "circleci" + case slices.Contains(env, "TRAVIS=true"): + return "travis" + case slices.Contains(env, "BUILDKITE=true"): + return "buildkite" + case slices.ContainsFunc(env, func(value string) bool { return strings.HasPrefix(value, "JENKINS_HOME=") }): + return "jenkins" + default: + return "ci" + } +} + +func verifyDomain(rawURL string) error { + u, err := url.Parse(rawURL) + if err != nil { + return errors.Wrap(err, "could not parse url") + } + + valid := []string{ + "localhost", + "docker.io", + "docker.com", + } + host := u.Hostname() + if !strings.ContainsRune(host, '.') { + return nil + } + for _, valid := range valid { + if host == valid { + return nil + } + if strings.HasSuffix(host, "."+valid) { + return nil + } + } + return errors.Errorf("invalid domain %s", host) +} diff --git a/driver/cloud/factory_test.go b/driver/cloud/factory_test.go new file mode 100644 index 000000000000..fc656c4617a7 --- /dev/null +++ b/driver/cloud/factory_test.go @@ -0,0 +1,100 @@ +package cloud + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTokenRefresh(t *testing.T) { + t.Parallel() + + now := time.Now() + count := 0 + s := func(context.Context) (string, error) { + count++ + return fmt.Sprintf("token-%d", count), nil + } + nextToken := newTokenSource(s, func() time.Time { return now }) + + token, err := nextToken(t.Context()) + require.NoError(t, err) + assert.Equal(t, "token-1", token) + + token, err = nextToken(t.Context()) + require.NoError(t, err) + assert.Equal(t, "token-1", token) + + now = now.Add(10 * time.Minute) + token, err = nextToken(t.Context()) + require.NoError(t, err) + assert.Equal(t, "token-2", token) +} + +func TestGetUserContext(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + expected string + env []string + }{ + { + name: "empty env", + env: []string{}, + expected: "local", + }, + { + name: "CI env var must be set or fallback to local", + env: []string{"CIRCLECI=true"}, + expected: "local", + }, + { + name: "unknown CI", + env: []string{"CI=true"}, + expected: "ci", + }, + { + name: "github ci detection", + env: []string{"CI=true", "GITHUB_ACTIONS=true"}, + expected: "github", + }, + { + name: "gitlab ci detection", + env: []string{"CI=true", "GITLAB_CI=true"}, + expected: "gitlab", + }, + { + name: "circleci ci detection", + env: []string{"CI=true", "CIRCLECI=true"}, + expected: "circleci", + }, + { + name: "travis ci detection", + env: []string{"CI=true", "TRAVIS=true"}, + expected: "travis", + }, + { + name: "buildkite ci detection", + env: []string{"CI=true", "BUILDKITE=true"}, + expected: "buildkite", + }, + { + name: "jenkins ci detection", + env: []string{"CI=true", "JENKINS_HOME=/root/jenkins"}, + expected: "jenkins", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + userContext := getUserContext(tt.env) + assert.Equal(t, tt.expected, userContext) + }) + } +} diff --git a/driver/cloud/login.go b/driver/cloud/login.go new file mode 100644 index 000000000000..140ae447e32f --- /dev/null +++ b/driver/cloud/login.go @@ -0,0 +1,241 @@ +package cloud + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "sync" + "time" + + "github.com/docker/cli/cli/config" + "github.com/docker/cli/cli/config/types" + "github.com/pkg/errors" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel/propagation" +) + +type Token struct { + AccessToken string `json:"access_token"` +} + +// login user to cloud build service and retrieve a build token. +func login(ctx context.Context, registryHostname, host, builder string) (string, error) { + username, secret, err := getUserCreds(registryHostname) + if err != nil { + return "", errors.Wrap(err, "retrieving user credentials") + } + + // Retrieve Auth token with build scope. + scope := fmt.Sprintf("scope=builder:%s:build", builder) + // TODO silvin: use an env var to override the auth host. + authURL := fmt.Sprintf("%s/token?%s&%s", host, service, scope) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, authURL, nil) + if err != nil { + return "", errors.Wrap(err, "failed to create request") + } + req.SetBasicAuth(username, secret) + resp, err := newClient(nil).Do(req) + if err != nil || resp.StatusCode != http.StatusOK { + if resp == nil { + return "", errors.Wrap(err, "failed to get token") + } + defer resp.Body.Close() + // Pull the first 255 chars of the response body to help with debugging. + body, _ := io.ReadAll(io.LimitReader(resp.Body, 255)) + return "", errors.Errorf("got %s. Trace ID: %s. Response body: %s", resp.Status, + resp.Header.Get("x-trace-id"), string(body)) + } + + defer resp.Body.Close() + var token Token + if err := json.NewDecoder(resp.Body).Decode(&token); err != nil { + return "", errors.Wrap(err, "failed to decode token") + } + return token.AccessToken, nil +} + +// fetchHubToken retrieves a token from Hub that we can use to call Hub APIs. +func fetchHubToken(ctx context.Context, registryHostname, hubHost string) (string, error) { + username, secret, err := getUserCreds(registryHostname) + if err != nil { + return "", errors.Wrap(err, "retrieving user credentials") + } + + hubURL, err := url.Parse(hubHost) + if err != nil { + return "", errors.Wrap(err, "invalid Docker Hub URL") + } + tokenURL, err := hubURL.Parse("v2/auth/token") + if err != nil { + return "", errors.Wrap(err, "invalid token URL") + } + var payload struct { + Identifier string `json:"identifier"` + Secret string `json:"secret"` + } + payload.Identifier = username + payload.Secret = secret + body, err := json.Marshal(payload) + if err != nil { + return "", errors.Wrap(err, "failed to marshal request body") + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, tokenURL.String(), bytes.NewReader(body)) + if err != nil { + return "", errors.Wrap(err, "create request") + } + req.Header.Set("Content-Type", "application/json") + + resp, err := newClient(nil).Do(req) + if err != nil { + return "", errors.Wrap(err, "get token") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + // Pull the first 255 chars of the response body to help with debugging. + body, _ := io.ReadAll(io.LimitReader(resp.Body, 255)) + return "", errors.Errorf("failed to get token, got: %s. Trace ID: %s. Response body: %s", resp.Status, + resp.Header.Get("x-trace-id"), string(body)) + } + + var respPayload struct { + Token string `json:"access_token"` + } + if err := json.NewDecoder(resp.Body).Decode(&respPayload); err != nil { + return "", errors.Wrap(err, "decode token") + } + return respPayload.Token, nil +} + +// getUserCreds retrieves user credentials from the credential store. +func getUserCreds(registryHostname string) (string, string, error) { + dockerConfig := config.LoadDefaultConfigFile(os.Stderr) + authConfig, err := dockerConfig.GetAuthConfig(registryHostname) + if err != nil { + return "", "", errors.Wrap(err, "retrieving auth config") + } + // If the store has no credentials, it will return a zero value and no + // error, eventually failing with a difficult to understand error later. + // Make it explicit here. + if authConfig == (types.AuthConfig{}) { + return "", "", errors.Errorf("no credentials found for %s", registryHostname) + } + username := authConfig.Username + secret := authConfig.Password + if authConfig.IdentityToken != "" { + secret = authConfig.IdentityToken + } + + return username, secret, nil +} + +const ( + retryWaitInitial = 100 * time.Millisecond + retryWaitMax = 10 * time.Second + retryMax = 5 +) + +type retryClient struct { + client *http.Client +} + +func newClient(transport http.RoundTripper) *retryClient { + if transport == nil { + transport = http.DefaultTransport + } + propagators := otelhttp.WithPropagators( + propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}), + ) + return &retryClient{client: &http.Client{ + Transport: otelhttp.NewTransport(transport, propagators), + }} +} + +func (c *retryClient) Do(req *http.Request) (*http.Response, error) { + for attempt := 0; ; attempt++ { + attemptReq := req.Clone(req.Context()) + if attempt > 0 && req.Body != nil { + if req.GetBody == nil { + return nil, errors.New("request body cannot be replayed") + } + body, err := req.GetBody() + if err != nil { + return nil, err + } + attemptReq.Body = body + } + + resp, err := c.client.Do(attemptReq) + if !shouldRetry(req.Context(), resp, err) { + return resp, err + } + if attempt == retryMax { + if resp != nil { + drainResponse(resp) + } + if err != nil { + return nil, errors.Wrapf(err, "%s %s giving up after %d attempts", req.Method, req.URL, attempt+1) + } + return nil, errors.Errorf("%s %s giving up after %d attempts", req.Method, req.URL, attempt+1) + } + if resp != nil { + drainResponse(resp) + } + + wait := min(retryWaitInitial<= 500 && resp.StatusCode != http.StatusNotImplemented) +} + +func drainResponse(resp *http.Response) { + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) +} + +const tokenRefreshInterval = time.Minute + +type tokenSource func(context.Context) (string, error) + +// nextTokenFunc refreshes the token every minute. +// +// TODO(nicks): Unpack the token and look at its expiry to determine when to refresh it. +func newTokenSource(fetch func(context.Context) (string, error), now func() time.Time) tokenSource { + var mu sync.Mutex + var token string + var err error + var lastRefresh time.Time + return tokenSource(func(ctx context.Context) (string, error) { + mu.Lock() + defer mu.Unlock() + + if lastRefresh.IsZero() || now().Sub(lastRefresh) > tokenRefreshInterval { + lastRefresh = now() + ctx, cancel := context.WithTimeoutCause(ctx, 30*time.Second, errors.WithStack(context.DeadlineExceeded)) + defer cancel() + token, err = fetch(ctx) + } + return token, err + }) +} diff --git a/driver/cloud/login_test.go b/driver/cloud/login_test.go new file mode 100644 index 000000000000..c3f7263f7a59 --- /dev/null +++ b/driver/cloud/login_test.go @@ -0,0 +1,57 @@ +package cloud + +import ( + "bytes" + "io" + "net/http" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type roundTripperFunc func(*http.Request) (*http.Response, error) + +func (f roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} + +func TestRetryClient(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + var attempts int + transport := roundTripperFunc(func(req *http.Request) (*http.Response, error) { + attempts++ + body, err := io.ReadAll(req.Body) + require.NoError(t, err) + assert.Equal(t, "payload", string(body)) + + statusCode := http.StatusServiceUnavailable + if attempts == 3 { + statusCode = http.StatusOK + } + return &http.Response{ + StatusCode: statusCode, + Header: make(http.Header), + Body: http.NoBody, + }, nil + }) + + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://example.com", bytes.NewBufferString("payload")) + require.NoError(t, err) + + start := time.Now() + resp, err := newClient(transport).Do(req) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, 3, attempts) + // The first two failed attempts wait for the initial delay and twice + // that delay before the third attempt succeeds. + wantWait := retryWaitInitial + 2*retryWaitInitial + assert.Equal(t, wantWait, time.Since(start)) + }) +} From 9d0f3b2cea60174c54f3679dff318abc214fd065 Mon Sep 17 00:00:00 2001 From: Alberto Garcia Hierro Date: Wed, 22 Jul 2026 23:23:34 +0100 Subject: [PATCH 02/21] cloud: pull image during export finalization Register cloud pull through BuildKit's session exporter so it runs while result refs are still retained. This avoids relying on post-solve metadata after the daemon may have garbage-collected blobs. Pin BuildKit to the OSS callback commit based on v0.31.0. Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> Co-authored-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- build/build.go | 16 +- build/opt.go | 46 +- driver/cloud/driver.go | 37 +- .../buildkit/session/exporter/exporter.pb.go | 343 +++++ .../buildkit/session/exporter/exporter.proto | 30 + .../session/exporter/exporter_grpc.pb.go | 157 ++ .../session/exporter/exporter_vtproto.pb.go | 1358 +++++++++++++++++ .../exporter/exporterprovider/provider.go | 62 + vendor/modules.txt | 2 + 9 files changed, 2014 insertions(+), 37 deletions(-) create mode 100644 vendor/github.com/moby/buildkit/session/exporter/exporter.pb.go create mode 100644 vendor/github.com/moby/buildkit/session/exporter/exporter.proto create mode 100644 vendor/github.com/moby/buildkit/session/exporter/exporter_grpc.pb.go create mode 100644 vendor/github.com/moby/buildkit/session/exporter/exporter_vtproto.pb.go create mode 100644 vendor/github.com/moby/buildkit/session/exporter/exporterprovider/provider.go diff --git a/build/build.go b/build/build.go index 0c8253979cde..6abb7d98a1c6 100644 --- a/build/build.go +++ b/build/build.go @@ -98,9 +98,6 @@ type Options struct { GroupRef string Annotations map[exptypes.AnnotationKey]string // Not used during build, annotations are already set in Exports. Just used to check for support with drivers. Policy []buildflags.PolicyConfig - // onSolveResponse runs after BuildKit returns exporter metadata. Cloud pull - // uses it to import the image without a private BuildKit session exporter. - onSolveResponse func(context.Context, *client.SolveResponse) error } // ResourceLimits holds the cgroup resource constraints applied to individual @@ -235,8 +232,7 @@ type NamedContext struct { type reqForNode struct { *noderesolver.ResolvedNode - so *client.SolveOpt - onSolveResponse func(context.Context, *client.SolveResponse) error + so *client.SolveOpt } func filterAvailableNodes(nodes []builder.Node) ([]builder.Node, error) { @@ -357,9 +353,8 @@ func newBuildRequests(ctx context.Context, docker *dockerutil.Client, cfg *confu } addGitAttrs(so) reqn = append(reqn, &reqForNode{ - ResolvedNode: np, - so: so, - onSolveResponse: localOpt.onSolveResponse, + ResolvedNode: np, + so: so, }) } reqForNodes[k] = reqn @@ -746,11 +741,6 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[ for k, v := range callRes { rr.ExporterResponse[k] = string(v) } - if onSolveResponse := reqForNodes[k][i].onSolveResponse; onSolveResponse != nil { - if err := onSolveResponse(ctx, rr); err != nil { - return err - } - } if opt.CallFunc == nil { rr.ExporterResponse["buildx.build.ref"] = buildRef if node.Driver.HistoryAPISupported(ctx) { diff --git a/build/opt.go b/build/opt.go index cf9f03d693fd..32da70f8e152 100644 --- a/build/opt.go +++ b/build/opt.go @@ -42,6 +42,7 @@ import ( gateway "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/session" + "github.com/moby/buildkit/session/exporter/exporterprovider" "github.com/moby/buildkit/session/secrets/secretsprovider" "github.com/moby/buildkit/session/sshforward/sshprovider" "github.com/moby/buildkit/session/upload/uploadprovider" @@ -456,6 +457,7 @@ func toSolveOpt(ctx context.Context, np *noderesolver.ResolvedNode, multiDriver } // set up exporters + cloudPullConfigured := false for i, e := range opt.Exports { if e.Type == "oci" && !nodeDriver.Features(ctx)[driver.OCIExporter] { return nil, nil, notSupported(driver.OCIExporter, nodeDriver, "https://docs.docker.com/go/build-exporters/") @@ -526,25 +528,31 @@ func toSolveOpt(ctx context.Context, np *noderesolver.ResolvedNode, multiDriver if len(opt.Platforms) > 1 { return nil, nil, errors.Errorf("cloud pull for single node multi-platform builds currently not supported") } - cloudDriver := node.Driver.Driver.(*cloud.Driver) - targetPlatforms := slices.Clone(opt.Platforms) - tags := slices.Clone(opt.Tags) - opt.onSolveResponse = func(ctx context.Context, resp *client.SolveResponse) error { - pw := progress.ResetTime(pw) - return progress.Wrap("cloud pull", pw.Write, func(l progress.SubLogger) error { - imgDescriptor := resp.ExporterResponse[exptypes.ExporterImageDescriptorKey] - platform := resp.ExporterResponse[exptypes.ExporterPlatformsKey] - // Fallback: if platform is not in buildkit response - // but --platform was specified, use it - if platform == "" && len(targetPlatforms) == 1 { - platform = platforms.Format(targetPlatforms[0]) - } - - if err := cloudDriver.CloudPull(ctx, imgDescriptor, platform, docker, tags, l); err != nil { - return errors.Wrap(err, "pulling image from cloud") - } - return nil - }) + if !cloudPullConfigured { + cloudPullConfigured = true + cloudDriver := node.Driver.Driver.(*cloud.Driver) + targetPlatforms := slices.Clone(opt.Platforms) + tags := slices.Clone(opt.Tags) + dockerContext := e.Attrs["context"] + callback := exporterprovider.New(nil, exporterprovider.WithFinalizeCallback(func(ctx context.Context, exporterResponse map[string]string) error { + pw := progress.ResetTime(pw) + return progress.Wrap("cloud pull", pw.Write, func(l progress.SubLogger) error { + imgDescriptor := exporterResponse[exptypes.ExporterImageDescriptorKey] + platform := exporterResponse[exptypes.ExporterPlatformsKey] + // Fallback: if platform is not in buildkit response + // but --platform was specified, use it + if platform == "" && len(targetPlatforms) == 1 { + platform = platforms.Format(targetPlatforms[0]) + } + + if err := cloudDriver.CloudPull(ctx, imgDescriptor, platform, docker, dockerContext, tags, l); err != nil { + return errors.Wrap(err, "pulling image from cloud") + } + return nil + }) + })) + opt.Session = append(opt.Session, callback) + so.EnableSessionExporter = true } } } diff --git a/driver/cloud/driver.go b/driver/cloud/driver.go index 0ae0443dee7b..34d86001285a 100644 --- a/driver/cloud/driver.go +++ b/driver/cloud/driver.go @@ -194,8 +194,8 @@ func (d *Driver) HostGatewayIP(ctx context.Context) (net.IP, error) { // CloudPull pulls an image from the cloud registry. // It should be called right after Solve returns. -func (d *Driver) CloudPull(ctx context.Context, imageDescriptor string, platform string, duc *dockerutil.Client, tags []string, l progress.SubLogger) error { - dc, err := duc.API("") +func (d *Driver) CloudPull(ctx context.Context, imageDescriptor string, platform string, duc *dockerutil.Client, dockerContext string, tags []string, l progress.SubLogger) error { + dc, err := duc.API(dockerContext) if err != nil { return err } @@ -286,7 +286,31 @@ func (d *Driver) CheckCloudPull(ctx context.Context, docker *dockerutil.Client, return exports } - supportsRegistryTokenPull, err := daemonSupportsRegistryTokenPull(ctx, docker) + dockerContext := "" + hasDockerContext := false + hasCloudPull := len(exports) == 0 + for _, e := range exports { + pushAttr, _ := strconv.ParseBool(e.Attrs["push"]) + if e.Type != "docker" && (e.Type != "image" || pushAttr) { + continue + } + if e.Output != nil || e.OutputDir != "" { + continue + } + hasCloudPull = true + context := e.Attrs["context"] + if !hasDockerContext { + dockerContext = context + hasDockerContext = true + } else if dockerContext != context { + return exports + } + } + if !hasCloudPull { + return exports + } + + supportsRegistryTokenPull, err := daemonSupportsRegistryTokenPull(ctx, docker, dockerContext) if err != nil { logrus.Warnf("failed to detect registry pull token support in daemon: %v", err) return exports @@ -308,6 +332,9 @@ func (d *Driver) CheckCloudPull(ctx context.Context, docker *dockerutil.Client, for i, e := range exports { pushAttr, _ := strconv.ParseBool(e.Attrs["push"]) + if e.Output != nil || e.OutputDir != "" { + continue + } if e.Type == "docker" || (e.Type == "image" && !pushAttr) { exports[i].Type = CloudPullExportType maps.Copy(exports[i].Attrs, CouldPullExportRequiredAttributes) @@ -424,8 +451,8 @@ func printMagicPull(rc io.Reader, l progress.SubLogger) (err error) { } } -func daemonSupportsRegistryTokenPull(ctx context.Context, duc *dockerutil.Client) (bool, error) { - dc, err := duc.API("") +func daemonSupportsRegistryTokenPull(ctx context.Context, duc *dockerutil.Client, dockerContext string) (bool, error) { + dc, err := duc.API(dockerContext) if err != nil { return false, err } diff --git a/vendor/github.com/moby/buildkit/session/exporter/exporter.pb.go b/vendor/github.com/moby/buildkit/session/exporter/exporter.pb.go new file mode 100644 index 000000000000..6b25e780ee1d --- /dev/null +++ b/vendor/github.com/moby/buildkit/session/exporter/exporter.pb.go @@ -0,0 +1,343 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11-devel +// protoc v3.14.0 +// source: github.com/moby/buildkit/session/exporter/exporter.proto + +package exporter + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type FindExportersRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Metadata map[string][]byte `protobuf:"bytes,1,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Refs []string `protobuf:"bytes,2,rep,name=refs,proto3" json:"refs,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FindExportersRequest) Reset() { + *x = FindExportersRequest{} + mi := &file_github_com_moby_buildkit_session_exporter_exporter_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FindExportersRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FindExportersRequest) ProtoMessage() {} + +func (x *FindExportersRequest) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_session_exporter_exporter_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FindExportersRequest.ProtoReflect.Descriptor instead. +func (*FindExportersRequest) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_session_exporter_exporter_proto_rawDescGZIP(), []int{0} +} + +func (x *FindExportersRequest) GetMetadata() map[string][]byte { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *FindExportersRequest) GetRefs() []string { + if x != nil { + return x.Refs + } + return nil +} + +type FindExportersResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Exporters []*ExporterRequest `protobuf:"bytes,1,rep,name=exporters,proto3" json:"exporters,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FindExportersResponse) Reset() { + *x = FindExportersResponse{} + mi := &file_github_com_moby_buildkit_session_exporter_exporter_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FindExportersResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FindExportersResponse) ProtoMessage() {} + +func (x *FindExportersResponse) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_session_exporter_exporter_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FindExportersResponse.ProtoReflect.Descriptor instead. +func (*FindExportersResponse) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_session_exporter_exporter_proto_rawDescGZIP(), []int{1} +} + +func (x *FindExportersResponse) GetExporters() []*ExporterRequest { + if x != nil { + return x.Exporters + } + return nil +} + +type ExporterRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Type string `protobuf:"bytes,1,opt,name=Type,proto3" json:"Type,omitempty"` + Attrs map[string]string `protobuf:"bytes,2,rep,name=Attrs,proto3" json:"Attrs,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExporterRequest) Reset() { + *x = ExporterRequest{} + mi := &file_github_com_moby_buildkit_session_exporter_exporter_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExporterRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExporterRequest) ProtoMessage() {} + +func (x *ExporterRequest) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_session_exporter_exporter_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExporterRequest.ProtoReflect.Descriptor instead. +func (*ExporterRequest) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_session_exporter_exporter_proto_rawDescGZIP(), []int{2} +} + +func (x *ExporterRequest) GetType() string { + if x != nil { + return x.Type + } + return "" +} + +func (x *ExporterRequest) GetAttrs() map[string]string { + if x != nil { + return x.Attrs + } + return nil +} + +type FinalizeExportRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + ExporterResponse map[string]string `protobuf:"bytes,1,rep,name=exporter_response,json=exporterResponse,proto3" json:"exporter_response,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FinalizeExportRequest) Reset() { + *x = FinalizeExportRequest{} + mi := &file_github_com_moby_buildkit_session_exporter_exporter_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FinalizeExportRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FinalizeExportRequest) ProtoMessage() {} + +func (x *FinalizeExportRequest) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_session_exporter_exporter_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FinalizeExportRequest.ProtoReflect.Descriptor instead. +func (*FinalizeExportRequest) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_session_exporter_exporter_proto_rawDescGZIP(), []int{3} +} + +func (x *FinalizeExportRequest) GetExporterResponse() map[string]string { + if x != nil { + return x.ExporterResponse + } + return nil +} + +type FinalizeExportResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FinalizeExportResponse) Reset() { + *x = FinalizeExportResponse{} + mi := &file_github_com_moby_buildkit_session_exporter_exporter_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FinalizeExportResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FinalizeExportResponse) ProtoMessage() {} + +func (x *FinalizeExportResponse) ProtoReflect() protoreflect.Message { + mi := &file_github_com_moby_buildkit_session_exporter_exporter_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FinalizeExportResponse.ProtoReflect.Descriptor instead. +func (*FinalizeExportResponse) Descriptor() ([]byte, []int) { + return file_github_com_moby_buildkit_session_exporter_exporter_proto_rawDescGZIP(), []int{4} +} + +var File_github_com_moby_buildkit_session_exporter_exporter_proto protoreflect.FileDescriptor + +const file_github_com_moby_buildkit_session_exporter_exporter_proto_rawDesc = "" + + "\n" + + "8github.com/moby/buildkit/session/exporter/exporter.proto\x12\x10moby.exporter.v1\"\xb9\x01\n" + + "\x14FindExportersRequest\x12P\n" + + "\bmetadata\x18\x01 \x03(\v24.moby.exporter.v1.FindExportersRequest.MetadataEntryR\bmetadata\x12\x12\n" + + "\x04refs\x18\x02 \x03(\tR\x04refs\x1a;\n" + + "\rMetadataEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\fR\x05value:\x028\x01\"X\n" + + "\x15FindExportersResponse\x12?\n" + + "\texporters\x18\x01 \x03(\v2!.moby.exporter.v1.ExporterRequestR\texporters\"\xa3\x01\n" + + "\x0fExporterRequest\x12\x12\n" + + "\x04Type\x18\x01 \x01(\tR\x04Type\x12B\n" + + "\x05Attrs\x18\x02 \x03(\v2,.moby.exporter.v1.ExporterRequest.AttrsEntryR\x05Attrs\x1a8\n" + + "\n" + + "AttrsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xc8\x01\n" + + "\x15FinalizeExportRequest\x12j\n" + + "\x11exporter_response\x18\x01 \x03(\v2=.moby.exporter.v1.FinalizeExportRequest.ExporterResponseEntryR\x10exporterResponse\x1aC\n" + + "\x15ExporterResponseEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x18\n" + + "\x16FinalizeExportResponse2\xd1\x01\n" + + "\bExporter\x12`\n" + + "\rFindExporters\x12&.moby.exporter.v1.FindExportersRequest\x1a'.moby.exporter.v1.FindExportersResponse\x12c\n" + + "\x0eFinalizeExport\x12'.moby.exporter.v1.FinalizeExportRequest\x1a(.moby.exporter.v1.FinalizeExportResponseB+Z)github.com/moby/buildkit/session/exporterb\x06proto3" + +var ( + file_github_com_moby_buildkit_session_exporter_exporter_proto_rawDescOnce sync.Once + file_github_com_moby_buildkit_session_exporter_exporter_proto_rawDescData []byte +) + +func file_github_com_moby_buildkit_session_exporter_exporter_proto_rawDescGZIP() []byte { + file_github_com_moby_buildkit_session_exporter_exporter_proto_rawDescOnce.Do(func() { + file_github_com_moby_buildkit_session_exporter_exporter_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_github_com_moby_buildkit_session_exporter_exporter_proto_rawDesc), len(file_github_com_moby_buildkit_session_exporter_exporter_proto_rawDesc))) + }) + return file_github_com_moby_buildkit_session_exporter_exporter_proto_rawDescData +} + +var file_github_com_moby_buildkit_session_exporter_exporter_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_github_com_moby_buildkit_session_exporter_exporter_proto_goTypes = []any{ + (*FindExportersRequest)(nil), // 0: moby.exporter.v1.FindExportersRequest + (*FindExportersResponse)(nil), // 1: moby.exporter.v1.FindExportersResponse + (*ExporterRequest)(nil), // 2: moby.exporter.v1.ExporterRequest + (*FinalizeExportRequest)(nil), // 3: moby.exporter.v1.FinalizeExportRequest + (*FinalizeExportResponse)(nil), // 4: moby.exporter.v1.FinalizeExportResponse + nil, // 5: moby.exporter.v1.FindExportersRequest.MetadataEntry + nil, // 6: moby.exporter.v1.ExporterRequest.AttrsEntry + nil, // 7: moby.exporter.v1.FinalizeExportRequest.ExporterResponseEntry +} +var file_github_com_moby_buildkit_session_exporter_exporter_proto_depIdxs = []int32{ + 5, // 0: moby.exporter.v1.FindExportersRequest.metadata:type_name -> moby.exporter.v1.FindExportersRequest.MetadataEntry + 2, // 1: moby.exporter.v1.FindExportersResponse.exporters:type_name -> moby.exporter.v1.ExporterRequest + 6, // 2: moby.exporter.v1.ExporterRequest.Attrs:type_name -> moby.exporter.v1.ExporterRequest.AttrsEntry + 7, // 3: moby.exporter.v1.FinalizeExportRequest.exporter_response:type_name -> moby.exporter.v1.FinalizeExportRequest.ExporterResponseEntry + 0, // 4: moby.exporter.v1.Exporter.FindExporters:input_type -> moby.exporter.v1.FindExportersRequest + 3, // 5: moby.exporter.v1.Exporter.FinalizeExport:input_type -> moby.exporter.v1.FinalizeExportRequest + 1, // 6: moby.exporter.v1.Exporter.FindExporters:output_type -> moby.exporter.v1.FindExportersResponse + 4, // 7: moby.exporter.v1.Exporter.FinalizeExport:output_type -> moby.exporter.v1.FinalizeExportResponse + 6, // [6:8] is the sub-list for method output_type + 4, // [4:6] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name +} + +func init() { file_github_com_moby_buildkit_session_exporter_exporter_proto_init() } +func file_github_com_moby_buildkit_session_exporter_exporter_proto_init() { + if File_github_com_moby_buildkit_session_exporter_exporter_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_github_com_moby_buildkit_session_exporter_exporter_proto_rawDesc), len(file_github_com_moby_buildkit_session_exporter_exporter_proto_rawDesc)), + NumEnums: 0, + NumMessages: 8, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_github_com_moby_buildkit_session_exporter_exporter_proto_goTypes, + DependencyIndexes: file_github_com_moby_buildkit_session_exporter_exporter_proto_depIdxs, + MessageInfos: file_github_com_moby_buildkit_session_exporter_exporter_proto_msgTypes, + }.Build() + File_github_com_moby_buildkit_session_exporter_exporter_proto = out.File + file_github_com_moby_buildkit_session_exporter_exporter_proto_goTypes = nil + file_github_com_moby_buildkit_session_exporter_exporter_proto_depIdxs = nil +} diff --git a/vendor/github.com/moby/buildkit/session/exporter/exporter.proto b/vendor/github.com/moby/buildkit/session/exporter/exporter.proto new file mode 100644 index 000000000000..76f4e1c9d675 --- /dev/null +++ b/vendor/github.com/moby/buildkit/session/exporter/exporter.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package moby.exporter.v1; + +option go_package = "github.com/moby/buildkit/session/exporter"; + +service Exporter { + rpc FindExporters(FindExportersRequest) returns (FindExportersResponse); + rpc FinalizeExport(FinalizeExportRequest) returns (FinalizeExportResponse); +} + +message FindExportersRequest{ + map metadata = 1; + repeated string refs = 2; +} + +message FindExportersResponse { + repeated ExporterRequest exporters = 1; +} + +message ExporterRequest { + string Type = 1; + map Attrs = 2; +} + +message FinalizeExportRequest { + map exporter_response = 1; +} + +message FinalizeExportResponse {} diff --git a/vendor/github.com/moby/buildkit/session/exporter/exporter_grpc.pb.go b/vendor/github.com/moby/buildkit/session/exporter/exporter_grpc.pb.go new file mode 100644 index 000000000000..af1559be5a32 --- /dev/null +++ b/vendor/github.com/moby/buildkit/session/exporter/exporter_grpc.pb.go @@ -0,0 +1,157 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.1 +// - protoc v3.14.0 +// source: github.com/moby/buildkit/session/exporter/exporter.proto + +package exporter + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Exporter_FindExporters_FullMethodName = "/moby.exporter.v1.Exporter/FindExporters" + Exporter_FinalizeExport_FullMethodName = "/moby.exporter.v1.Exporter/FinalizeExport" +) + +// ExporterClient is the client API for Exporter service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ExporterClient interface { + FindExporters(ctx context.Context, in *FindExportersRequest, opts ...grpc.CallOption) (*FindExportersResponse, error) + FinalizeExport(ctx context.Context, in *FinalizeExportRequest, opts ...grpc.CallOption) (*FinalizeExportResponse, error) +} + +type exporterClient struct { + cc grpc.ClientConnInterface +} + +func NewExporterClient(cc grpc.ClientConnInterface) ExporterClient { + return &exporterClient{cc} +} + +func (c *exporterClient) FindExporters(ctx context.Context, in *FindExportersRequest, opts ...grpc.CallOption) (*FindExportersResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(FindExportersResponse) + err := c.cc.Invoke(ctx, Exporter_FindExporters_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *exporterClient) FinalizeExport(ctx context.Context, in *FinalizeExportRequest, opts ...grpc.CallOption) (*FinalizeExportResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(FinalizeExportResponse) + err := c.cc.Invoke(ctx, Exporter_FinalizeExport_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ExporterServer is the server API for Exporter service. +// All implementations should embed UnimplementedExporterServer +// for forward compatibility. +type ExporterServer interface { + FindExporters(context.Context, *FindExportersRequest) (*FindExportersResponse, error) + FinalizeExport(context.Context, *FinalizeExportRequest) (*FinalizeExportResponse, error) +} + +// UnimplementedExporterServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedExporterServer struct{} + +func (UnimplementedExporterServer) FindExporters(context.Context, *FindExportersRequest) (*FindExportersResponse, error) { + return nil, status.Error(codes.Unimplemented, "method FindExporters not implemented") +} +func (UnimplementedExporterServer) FinalizeExport(context.Context, *FinalizeExportRequest) (*FinalizeExportResponse, error) { + return nil, status.Error(codes.Unimplemented, "method FinalizeExport not implemented") +} +func (UnimplementedExporterServer) testEmbeddedByValue() {} + +// UnsafeExporterServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ExporterServer will +// result in compilation errors. +type UnsafeExporterServer interface { + mustEmbedUnimplementedExporterServer() +} + +func RegisterExporterServer(s grpc.ServiceRegistrar, srv ExporterServer) { + // If the following call panics, it indicates UnimplementedExporterServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&Exporter_ServiceDesc, srv) +} + +func _Exporter_FindExporters_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(FindExportersRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExporterServer).FindExporters(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Exporter_FindExporters_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExporterServer).FindExporters(ctx, req.(*FindExportersRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Exporter_FinalizeExport_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(FinalizeExportRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExporterServer).FinalizeExport(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Exporter_FinalizeExport_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExporterServer).FinalizeExport(ctx, req.(*FinalizeExportRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// Exporter_ServiceDesc is the grpc.ServiceDesc for Exporter service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Exporter_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "moby.exporter.v1.Exporter", + HandlerType: (*ExporterServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "FindExporters", + Handler: _Exporter_FindExporters_Handler, + }, + { + MethodName: "FinalizeExport", + Handler: _Exporter_FinalizeExport_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "github.com/moby/buildkit/session/exporter/exporter.proto", +} diff --git a/vendor/github.com/moby/buildkit/session/exporter/exporter_vtproto.pb.go b/vendor/github.com/moby/buildkit/session/exporter/exporter_vtproto.pb.go new file mode 100644 index 000000000000..466c1f436961 --- /dev/null +++ b/vendor/github.com/moby/buildkit/session/exporter/exporter_vtproto.pb.go @@ -0,0 +1,1358 @@ +// Code generated by protoc-gen-go-vtproto. DO NOT EDIT. +// protoc-gen-go-vtproto version: v0.6.1-0.20240319094008-0393e58bdf10 +// source: github.com/moby/buildkit/session/exporter/exporter.proto + +package exporter + +import ( + fmt "fmt" + protohelpers "github.com/planetscale/vtprotobuf/protohelpers" + proto "google.golang.org/protobuf/proto" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + io "io" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +func (m *FindExportersRequest) CloneVT() *FindExportersRequest { + if m == nil { + return (*FindExportersRequest)(nil) + } + r := new(FindExportersRequest) + if rhs := m.Metadata; rhs != nil { + tmpContainer := make(map[string][]byte, len(rhs)) + for k, v := range rhs { + tmpBytes := make([]byte, len(v)) + copy(tmpBytes, v) + tmpContainer[k] = tmpBytes + } + r.Metadata = tmpContainer + } + if rhs := m.Refs; rhs != nil { + tmpContainer := make([]string, len(rhs)) + copy(tmpContainer, rhs) + r.Refs = tmpContainer + } + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *FindExportersRequest) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (m *FindExportersResponse) CloneVT() *FindExportersResponse { + if m == nil { + return (*FindExportersResponse)(nil) + } + r := new(FindExportersResponse) + if rhs := m.Exporters; rhs != nil { + tmpContainer := make([]*ExporterRequest, len(rhs)) + for k, v := range rhs { + tmpContainer[k] = v.CloneVT() + } + r.Exporters = tmpContainer + } + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *FindExportersResponse) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (m *ExporterRequest) CloneVT() *ExporterRequest { + if m == nil { + return (*ExporterRequest)(nil) + } + r := new(ExporterRequest) + r.Type = m.Type + if rhs := m.Attrs; rhs != nil { + tmpContainer := make(map[string]string, len(rhs)) + for k, v := range rhs { + tmpContainer[k] = v + } + r.Attrs = tmpContainer + } + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *ExporterRequest) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (m *FinalizeExportRequest) CloneVT() *FinalizeExportRequest { + if m == nil { + return (*FinalizeExportRequest)(nil) + } + r := new(FinalizeExportRequest) + if rhs := m.ExporterResponse; rhs != nil { + tmpContainer := make(map[string]string, len(rhs)) + for k, v := range rhs { + tmpContainer[k] = v + } + r.ExporterResponse = tmpContainer + } + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *FinalizeExportRequest) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (m *FinalizeExportResponse) CloneVT() *FinalizeExportResponse { + if m == nil { + return (*FinalizeExportResponse)(nil) + } + r := new(FinalizeExportResponse) + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *FinalizeExportResponse) CloneMessageVT() proto.Message { + return m.CloneVT() +} + +func (this *FindExportersRequest) EqualVT(that *FindExportersRequest) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if len(this.Metadata) != len(that.Metadata) { + return false + } + for i, vx := range this.Metadata { + vy, ok := that.Metadata[i] + if !ok { + return false + } + if string(vx) != string(vy) { + return false + } + } + if len(this.Refs) != len(that.Refs) { + return false + } + for i, vx := range this.Refs { + vy := that.Refs[i] + if vx != vy { + return false + } + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *FindExportersRequest) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*FindExportersRequest) + if !ok { + return false + } + return this.EqualVT(that) +} +func (this *FindExportersResponse) EqualVT(that *FindExportersResponse) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if len(this.Exporters) != len(that.Exporters) { + return false + } + for i, vx := range this.Exporters { + vy := that.Exporters[i] + if p, q := vx, vy; p != q { + if p == nil { + p = &ExporterRequest{} + } + if q == nil { + q = &ExporterRequest{} + } + if !p.EqualVT(q) { + return false + } + } + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *FindExportersResponse) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*FindExportersResponse) + if !ok { + return false + } + return this.EqualVT(that) +} +func (this *ExporterRequest) EqualVT(that *ExporterRequest) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if this.Type != that.Type { + return false + } + if len(this.Attrs) != len(that.Attrs) { + return false + } + for i, vx := range this.Attrs { + vy, ok := that.Attrs[i] + if !ok { + return false + } + if vx != vy { + return false + } + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *ExporterRequest) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*ExporterRequest) + if !ok { + return false + } + return this.EqualVT(that) +} +func (this *FinalizeExportRequest) EqualVT(that *FinalizeExportRequest) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if len(this.ExporterResponse) != len(that.ExporterResponse) { + return false + } + for i, vx := range this.ExporterResponse { + vy, ok := that.ExporterResponse[i] + if !ok { + return false + } + if vx != vy { + return false + } + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *FinalizeExportRequest) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*FinalizeExportRequest) + if !ok { + return false + } + return this.EqualVT(that) +} +func (this *FinalizeExportResponse) EqualVT(that *FinalizeExportResponse) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *FinalizeExportResponse) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*FinalizeExportResponse) + if !ok { + return false + } + return this.EqualVT(that) +} +func (m *FindExportersRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FindExportersRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FindExportersRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Refs) > 0 { + for iNdEx := len(m.Refs) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Refs[iNdEx]) + copy(dAtA[i:], m.Refs[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Refs[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.Metadata) > 0 { + for k := range m.Metadata { + v := m.Metadata[k] + baseI := i + i -= len(v) + copy(dAtA[i:], v) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(v))) + i-- + dAtA[i] = 0x12 + i -= len(k) + copy(dAtA[i:], k) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(k))) + i-- + dAtA[i] = 0xa + i = protohelpers.EncodeVarint(dAtA, i, uint64(baseI-i)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *FindExportersResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FindExportersResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FindExportersResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Exporters) > 0 { + for iNdEx := len(m.Exporters) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Exporters[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *ExporterRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ExporterRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *ExporterRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Attrs) > 0 { + for k := range m.Attrs { + v := m.Attrs[k] + baseI := i + i -= len(v) + copy(dAtA[i:], v) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(v))) + i-- + dAtA[i] = 0x12 + i -= len(k) + copy(dAtA[i:], k) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(k))) + i-- + dAtA[i] = 0xa + i = protohelpers.EncodeVarint(dAtA, i, uint64(baseI-i)) + i-- + dAtA[i] = 0x12 + } + } + if len(m.Type) > 0 { + i -= len(m.Type) + copy(dAtA[i:], m.Type) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Type))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *FinalizeExportRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FinalizeExportRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FinalizeExportRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.ExporterResponse) > 0 { + for k := range m.ExporterResponse { + v := m.ExporterResponse[k] + baseI := i + i -= len(v) + copy(dAtA[i:], v) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(v))) + i-- + dAtA[i] = 0x12 + i -= len(k) + copy(dAtA[i:], k) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(k))) + i-- + dAtA[i] = 0xa + i = protohelpers.EncodeVarint(dAtA, i, uint64(baseI-i)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *FinalizeExportResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FinalizeExportResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FinalizeExportResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + return len(dAtA) - i, nil +} + +func (m *FindExportersRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Metadata) > 0 { + for k, v := range m.Metadata { + _ = k + _ = v + l = 1 + len(v) + protohelpers.SizeOfVarint(uint64(len(v))) + mapEntrySize := 1 + len(k) + protohelpers.SizeOfVarint(uint64(len(k))) + l + n += mapEntrySize + 1 + protohelpers.SizeOfVarint(uint64(mapEntrySize)) + } + } + if len(m.Refs) > 0 { + for _, s := range m.Refs { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *FindExportersResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Exporters) > 0 { + for _, e := range m.Exporters { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *ExporterRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Type) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.Attrs) > 0 { + for k, v := range m.Attrs { + _ = k + _ = v + mapEntrySize := 1 + len(k) + protohelpers.SizeOfVarint(uint64(len(k))) + 1 + len(v) + protohelpers.SizeOfVarint(uint64(len(v))) + n += mapEntrySize + 1 + protohelpers.SizeOfVarint(uint64(mapEntrySize)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *FinalizeExportRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.ExporterResponse) > 0 { + for k, v := range m.ExporterResponse { + _ = k + _ = v + mapEntrySize := 1 + len(k) + protohelpers.SizeOfVarint(uint64(len(k))) + 1 + len(v) + protohelpers.SizeOfVarint(uint64(len(v))) + n += mapEntrySize + 1 + protohelpers.SizeOfVarint(uint64(mapEntrySize)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *FinalizeExportResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + n += len(m.unknownFields) + return n +} + +func (m *FindExportersRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FindExportersRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FindExportersRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Metadata", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Metadata == nil { + m.Metadata = make(map[string][]byte) + } + var mapkey string + var mapvalue []byte + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return protohelpers.ErrInvalidLength + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey < 0 { + return protohelpers.ErrInvalidLength + } + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var mapbyteLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapbyteLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intMapbyteLen := int(mapbyteLen) + if intMapbyteLen < 0 { + return protohelpers.ErrInvalidLength + } + postbytesIndex := iNdEx + intMapbyteLen + if postbytesIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postbytesIndex > l { + return io.ErrUnexpectedEOF + } + mapvalue = make([]byte, mapbyteLen) + copy(mapvalue, dAtA[iNdEx:postbytesIndex]) + iNdEx = postbytesIndex + } else { + iNdEx = entryPreIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.Metadata[mapkey] = mapvalue + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Refs", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Refs = append(m.Refs, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FindExportersResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FindExportersResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FindExportersResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Exporters", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Exporters = append(m.Exporters, &ExporterRequest{}) + if err := m.Exporters[len(m.Exporters)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ExporterRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ExporterRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ExporterRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Type = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Attrs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Attrs == nil { + m.Attrs = make(map[string]string) + } + var mapkey string + var mapvalue string + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return protohelpers.ErrInvalidLength + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey < 0 { + return protohelpers.ErrInvalidLength + } + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var stringLenmapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapvalue |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapvalue := int(stringLenmapvalue) + if intStringLenmapvalue < 0 { + return protohelpers.ErrInvalidLength + } + postStringIndexmapvalue := iNdEx + intStringLenmapvalue + if postStringIndexmapvalue < 0 { + return protohelpers.ErrInvalidLength + } + if postStringIndexmapvalue > l { + return io.ErrUnexpectedEOF + } + mapvalue = string(dAtA[iNdEx:postStringIndexmapvalue]) + iNdEx = postStringIndexmapvalue + } else { + iNdEx = entryPreIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.Attrs[mapkey] = mapvalue + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FinalizeExportRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FinalizeExportRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FinalizeExportRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ExporterResponse", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ExporterResponse == nil { + m.ExporterResponse = make(map[string]string) + } + var mapkey string + var mapvalue string + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return protohelpers.ErrInvalidLength + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey < 0 { + return protohelpers.ErrInvalidLength + } + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var stringLenmapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapvalue |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapvalue := int(stringLenmapvalue) + if intStringLenmapvalue < 0 { + return protohelpers.ErrInvalidLength + } + postStringIndexmapvalue := iNdEx + intStringLenmapvalue + if postStringIndexmapvalue < 0 { + return protohelpers.ErrInvalidLength + } + if postStringIndexmapvalue > l { + return io.ErrUnexpectedEOF + } + mapvalue = string(dAtA[iNdEx:postStringIndexmapvalue]) + iNdEx = postStringIndexmapvalue + } else { + iNdEx = entryPreIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.ExporterResponse[mapkey] = mapvalue + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FinalizeExportResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FinalizeExportResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FinalizeExportResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/vendor/github.com/moby/buildkit/session/exporter/exporterprovider/provider.go b/vendor/github.com/moby/buildkit/session/exporter/exporterprovider/provider.go new file mode 100644 index 000000000000..ac40d84fa2a9 --- /dev/null +++ b/vendor/github.com/moby/buildkit/session/exporter/exporterprovider/provider.go @@ -0,0 +1,62 @@ +package exporterprovider + +import ( + "context" + + "github.com/moby/buildkit/session/exporter" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type Callback func(ctx context.Context, md map[string][]byte, refs []string) ([]*exporter.ExporterRequest, error) + +type FinalizeCallback func(ctx context.Context, exporterResponse map[string]string) error + +type Option func(*Exporter) + +func WithFinalizeCallback(cb FinalizeCallback) Option { + return func(e *Exporter) { + e.finalize = cb + } +} + +func New(cb Callback, opts ...Option) *Exporter { + e := &Exporter{cb: cb} + for _, opt := range opts { + opt(e) + } + return e +} + +type Exporter struct { + cb Callback + finalize FinalizeCallback +} + +func (e *Exporter) FinalizeExport(ctx context.Context, in *exporter.FinalizeExportRequest) (*exporter.FinalizeExportResponse, error) { + if e.finalize == nil { + return nil, status.Errorf(codes.Unimplemented, "no exporter finalize callback registered") + } + if err := e.finalize(ctx, in.ExporterResponse); err != nil { + return nil, err + } + return &exporter.FinalizeExportResponse{}, nil +} + +func (e *Exporter) Register(server *grpc.Server) { + exporter.RegisterExporterServer(server, e) +} + +func (e *Exporter) FindExporters(ctx context.Context, in *exporter.FindExportersRequest) (*exporter.FindExportersResponse, error) { + if e.cb == nil { + return nil, status.Errorf(codes.Unavailable, "no exporter callback registered") + } + res, err := e.cb(ctx, in.Metadata, in.Refs) + if err != nil { + return nil, err + } + return &exporter.FindExportersResponse{ + Exporters: res, + }, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index d03de6cb8e8a..f5189c0950b4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -684,6 +684,8 @@ github.com/moby/buildkit/session github.com/moby/buildkit/session/auth github.com/moby/buildkit/session/auth/authprovider github.com/moby/buildkit/session/content +github.com/moby/buildkit/session/exporter +github.com/moby/buildkit/session/exporter/exporterprovider github.com/moby/buildkit/session/filesync github.com/moby/buildkit/session/grpchijack github.com/moby/buildkit/session/secrets From f0f0d870a5caaba5f2ae758c53773879c7740797 Mon Sep 17 00:00:00 2001 From: Alberto Garcia Hierro Date: Thu, 23 Jul 2026 12:56:26 +0100 Subject: [PATCH 03/21] builder: let factories name builders Allow driver factories to derive default builder names from endpoints before an instance is stored. Move cloud normalization into its factory. Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- builder/builder.go | 78 ++++++++++++++++++++++++------------ builder/builder_test.go | 56 ++++++++++++++++++++++++++ driver/cloud/factory.go | 6 +++ driver/cloud/factory_test.go | 12 ++++++ driver/manager.go | 18 ++++++++- 5 files changed, 142 insertions(+), 28 deletions(-) diff --git a/builder/builder.go b/builder/builder.go index 4be402837173..1e71b1230346 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -360,18 +360,7 @@ func Create(ctx context.Context, txn *store.Txn, dockerCli command.Cli, opts Cre } name := opts.Name - if name == "" { - if opts.Driver == cloud.DriverName && opts.Endpoint != "" { - name = "cloud-" + strings.ReplaceAll(strings.ToLower(opts.Endpoint), "/", "-") - } else { - name, err = store.GenerateName(txn) - if err != nil { - return nil, err - } - } - } - - if !opts.Append { + if name != "" && !opts.Append { contexts, err := dockerCli.ContextStore().List() if err != nil { return nil, err @@ -383,34 +372,75 @@ func Create(ctx context.Context, txn *store.Txn, dockerCli command.Cli, opts Cre } } - ng, err := txn.NodeGroupByName(name) - if err != nil { - if os.IsNotExist(errors.Cause(err)) { - if opts.Append && opts.Name != "" { - return nil, errors.Errorf("failed to find instance %q for append", opts.Name) + var ng *store.NodeGroup + if name != "" { + ng, err = txn.NodeGroupByName(name) + if err != nil { + if os.IsNotExist(errors.Cause(err)) { + if opts.Append { + return nil, errors.Errorf("failed to find instance %q for append", opts.Name) + } + } else { + return nil, err } - } else { - return nil, err } } buildkitHost := os.Getenv("BUILDKIT_HOST") driverName := opts.Driver + var factory driver.Factory if driverName == "" { if ng != nil { driverName = ng.Driver } else if opts.Endpoint == "" && buildkitHost != "" { driverName = "remote" } else { - f, err := driver.GetDefaultFactory(ctx, opts.Endpoint, dockerCli.Client(), true, nil) + factory, err = driver.GetDefaultFactory(ctx, opts.Endpoint, dockerCli.Client(), true, nil) if err != nil { return nil, err } - if f == nil { + if factory == nil { return nil, errors.Errorf("no valid drivers found") } - driverName = f.Name() + driverName = factory.Name() + } + } + + if factory == nil { + factory, err = driver.GetFactory(driverName, true) + if err != nil { + return nil, err + } + } + + if name == "" { + if namer, ok := factory.(driver.DefaultBuilderNamer); ok && opts.Endpoint != "" { + name, err = namer.DefaultBuilderName(ctx, opts.Endpoint) + if err != nil { + return nil, err + } + } + if name == "" { + name, err = store.GenerateName(txn) + if err != nil { + return nil, err + } + } + + contexts, err := dockerCli.ContextStore().List() + if err != nil { + return nil, err + } + for _, c := range contexts { + if c.Name == name { + return nil, errors.Errorf("instance name %q already exists as context builder", name) + } + } + + ng, err = txn.NodeGroupByName(name) + if err != nil && !os.IsNotExist(errors.Cause(err)) { + return nil, err } } @@ -423,10 +453,6 @@ func Create(ctx context.Context, txn *store.Txn, dockerCli command.Cli, opts Cre } } - if _, err := driver.GetFactory(driverName, true); err != nil { - return nil, err - } - ngOriginal := ng if ngOriginal != nil { ngOriginal = ngOriginal.Copy() diff --git a/builder/builder_test.go b/builder/builder_test.go index 6c3080683bea..dafe71577541 100644 --- a/builder/builder_test.go +++ b/builder/builder_test.go @@ -1,14 +1,70 @@ package builder import ( + "context" + "errors" "os" "path" "testing" + "github.com/docker/buildx/driver" + dockerclient "github.com/moby/moby/client" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +type testFactory struct { + name string +} + +func (f testFactory) Name() string { + return f.name +} + +func (f testFactory) Usage() string { + return f.name +} + +func (testFactory) Priority(context.Context, string, dockerclient.APIClient, map[string][]string) int { + return 10000 +} + +func (testFactory) New(context.Context, driver.InitConfig) (driver.Driver, error) { + return nil, errors.New("test factory cannot create drivers") +} + +func (testFactory) AllowsInstances() bool { + return true +} + +type defaultBuilderNamerFactory struct { + testFactory + endpoint string + err error +} + +func (f *defaultBuilderNamerFactory) DefaultBuilderName(_ context.Context, endpoint string) (string, error) { + f.endpoint = endpoint + return "", f.err +} + +func TestCreateDefaultBuilderNamer(t *testing.T) { + const endpoint = "org/builder" + expectedErr := errors.New("default builder name") + factory := &defaultBuilderNamerFactory{ + testFactory: testFactory{name: "test-default-builder-namer"}, + err: expectedErr, + } + t.Cleanup(driver.Register(factory)) + + _, err := Create(t.Context(), nil, nil, CreateOpts{ + Driver: factory.Name(), + Endpoint: endpoint, + }) + require.ErrorIs(t, err, expectedErr) + assert.Equal(t, endpoint, factory.endpoint) +} + func TestCsvToMap(t *testing.T) { d := []string{ "\"tolerations=key=foo,value=bar;key=foo2,value=bar2\",replicas=1", diff --git a/driver/cloud/factory.go b/driver/cloud/factory.go index 32f18a62e1d5..159f32d14411 100644 --- a/driver/cloud/factory.go +++ b/driver/cloud/factory.go @@ -72,6 +72,8 @@ type factory struct { once sync.Once } +var _ driver.DefaultBuilderNamer = (*factory)(nil) + func (*factory) Name() string { return DriverName } @@ -80,6 +82,10 @@ func (*factory) Usage() string { return DriverName } +func (*factory) DefaultBuilderName(_ context.Context, endpoint string) (string, error) { + return DriverName + "-" + strings.ReplaceAll(strings.ToLower(endpoint), "/", "-"), nil +} + func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient, _ map[string][]string) int { return lowPriority } diff --git a/driver/cloud/factory_test.go b/driver/cloud/factory_test.go index fc656c4617a7..a2995186c426 100644 --- a/driver/cloud/factory_test.go +++ b/driver/cloud/factory_test.go @@ -6,10 +6,22 @@ import ( "testing" "time" + "github.com/docker/buildx/driver" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func TestDefaultBuilderName(t *testing.T) { + t.Parallel() + + factory := &factory{} + namer, ok := any(factory).(driver.DefaultBuilderNamer) + require.True(t, ok) + name, err := namer.DefaultBuilderName(t.Context(), "Org/Builder") + require.NoError(t, err) + assert.Equal(t, "cloud-org-builder", name) +} + func TestTokenRefresh(t *testing.T) { t.Parallel() diff --git a/driver/manager.go b/driver/manager.go index b9ef2c275af5..db7f299f1b5a 100644 --- a/driver/manager.go +++ b/driver/manager.go @@ -24,6 +24,11 @@ type Factory interface { AllowsInstances() bool } +// DefaultBuilderNamer derives a default builder name from an endpoint. +type DefaultBuilderNamer interface { + DefaultBuilderName(ctx context.Context, endpoint string) (string, error) +} + type BuildkitConfig struct { // Entitlements []string // Rootless bool @@ -55,11 +60,20 @@ type InitConfig struct { var drivers map[string]Factory -func Register(f Factory) { +func Register(f Factory) func() { if drivers == nil { drivers = map[string]Factory{} } - drivers[f.Name()] = f + name := f.Name() + previous, ok := drivers[name] + drivers[name] = f + return func() { + if ok { + drivers[name] = previous + } else { + delete(drivers, name) + } + } } func GetDefaultFactory(ctx context.Context, ep string, c dockerclient.APIClient, instanceRequired bool, dialMeta map[string][]string) (Factory, error) { From 50d18c0f073375eb0a08d0e2db07dec954f7fc0c Mon Sep 17 00:00:00 2001 From: Alberto Garcia Hierro Date: Thu, 23 Jul 2026 14:09:45 +0100 Subject: [PATCH 04/21] builder: let drivers resolve nodes Co-authored-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- builder/builder.go | 80 +++++++++++-------- builder/builder_test.go | 155 ++++++++++++++++++++++++++++++++++++ driver/cloud/create.go | 45 ++++++++--- driver/cloud/create_test.go | 25 ++++-- driver/cloud/factory.go | 5 +- driver/manager.go | 12 +++ 6 files changed, 271 insertions(+), 51 deletions(-) diff --git a/builder/builder.go b/builder/builder.go index 1e71b1230346..98584877ebdf 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -12,7 +12,6 @@ import ( "time" "github.com/docker/buildx/driver" - "github.com/docker/buildx/driver/cloud" k8sutil "github.com/docker/buildx/driver/kubernetes/util" remoteutil "github.com/docker/buildx/driver/remote/util" "github.com/docker/buildx/localstate" @@ -350,6 +349,42 @@ type CreateOpts struct { Timeout time.Duration } +func updateNodeGroup(ctx context.Context, factory driver.Factory, ng *store.NodeGroup, node driver.Node, appendNode bool, buildkitdFlags []string, buildkitdConfigFile string) error { + var nodes []driver.Node + _, resolvesNodes := factory.(driver.NodeResolver) + if resolver, ok := factory.(driver.NodeResolver); ok { + var err error + nodes, err = resolver.ResolveNodes(ctx, node) + if err != nil { + return err + } + if len(nodes) == 0 { + return errors.Errorf("driver %q returned no nodes", factory.Name()) + } + } else { + nodes = []driver.Node{node} + } + + for i, node := range nodes { + appendResolvedNode := appendNode || i > 0 + if resolvesNodes && appendResolvedNode && node.Name != "" { + name, err := store.ValidateName(node.Name) + if err != nil { + return err + } + for _, existing := range ng.Nodes { + if existing.Name == name { + return errors.Errorf("node %q already exists", name) + } + } + } + if err := ng.Update(node.Name, node.Endpoint, node.Platforms, node.EndpointSet, appendResolvedNode, buildkitdFlags, buildkitdConfigFile, node.DriverOpts); err != nil { + return err + } + } + return nil +} + func Create(ctx context.Context, txn *store.Txn, dockerCli command.Cli, opts CreateOpts) (*Builder, error) { var err error @@ -486,6 +521,7 @@ func Create(ctx context.Context, txn *store.Txn, dockerCli command.Cli, opts Cre var ep string var setEp bool + _, resolvesNodes := factory.(driver.NodeResolver) switch { case driverName == "kubernetes": if opts.Endpoint != "" { @@ -523,12 +559,9 @@ func Create(ctx context.Context, txn *store.Txn, dockerCli command.Cli, opts Cre return nil, err } setEp = true - case driverName == cloud.DriverName: - if opts.Endpoint == "" { - return nil, errors.Errorf("no endpoint (builder) provided") - } - ep = cloud.EndpointPrefix + opts.Endpoint - setEp = false + case resolvesNodes: + // The factory resolves the supplied endpoint into concrete nodes. + ep = opts.Endpoint case opts.Endpoint != "": ep, err = validateEndpoint(dockerCli, opts.Endpoint) if err != nil { @@ -546,30 +579,15 @@ func Create(ctx context.Context, txn *store.Txn, dockerCli command.Cli, opts Cre setEp = false } - // support creating cloud builders by passing a builder group - // as the endpoint. This will create a node for each builder - // in the builder group. - // e.g. buildx create --driver cloud - if driverName == cloud.DriverName { - builders, err := cloud.GetBuilderInstances(ctx, opts.Endpoint, driverOpts) - if err != nil { - return nil, errors.Wrapf(err, "unknown builder") - } - for i, b := range builders { - if i != 0 { - opts.Append = true - } - - builderInstanceDriverOpts := cloud.GetBuilderInstanceDriverOpts(b, driverOpts) - if err := ng.Update(b.Name, b.Endpoint, []string{b.Platform}, false, opts.Append, buildkitdFlags, buildkitdConfigFile, builderInstanceDriverOpts); err != nil { - return nil, err - } - } - } else { - // default flow - if err := ng.Update(opts.NodeName, ep, opts.Platforms, setEp, opts.Append, buildkitdFlags, buildkitdConfigFile, driverOpts); err != nil { - return nil, err - } + node := driver.Node{ + Name: opts.NodeName, + Endpoint: ep, + Platforms: opts.Platforms, + EndpointSet: setEp, + DriverOpts: driverOpts, + } + if err := updateNodeGroup(ctx, factory, ng, node, opts.Append, buildkitdFlags, buildkitdConfigFile); err != nil { + return nil, err } if err := txn.Save(ng); err != nil { diff --git a/builder/builder_test.go b/builder/builder_test.go index dafe71577541..3d3e7944df25 100644 --- a/builder/builder_test.go +++ b/builder/builder_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/docker/buildx/driver" + "github.com/docker/buildx/store" dockerclient "github.com/moby/moby/client" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -37,6 +38,15 @@ func (testFactory) AllowsInstances() bool { return true } +type nodeResolvingFactory struct { + testFactory + nodes []driver.Node +} + +func (f nodeResolvingFactory) ResolveNodes(context.Context, driver.Node) ([]driver.Node, error) { + return f.nodes, nil +} + type defaultBuilderNamerFactory struct { testFactory endpoint string @@ -65,6 +75,151 @@ func TestCreateDefaultBuilderNamer(t *testing.T) { assert.Equal(t, endpoint, factory.endpoint) } +func TestUpdateNodeGroup(t *testing.T) { + t.Parallel() + + t.Run("Default", func(t *testing.T) { + t.Parallel() + + factory := testFactory{name: "example"} + ng := &store.NodeGroup{} + node := driver.Node{ + Name: "node", + Endpoint: "tcp://example:1234", + Platforms: []string{"linux/amd64"}, + EndpointSet: true, + DriverOpts: map[string]string{"key": "value"}, + } + + require.NoError(t, updateNodeGroup(t.Context(), factory, ng, node, false, nil, "")) + require.Len(t, ng.Nodes, 1) + assert.Equal(t, "node", ng.Nodes[0].Name) + assert.Equal(t, "tcp://example:1234", ng.Nodes[0].Endpoint) + assert.Equal(t, "amd64", ng.Nodes[0].Platforms[0].Architecture) + assert.Equal(t, map[string]string{"key": "value"}, ng.Nodes[0].DriverOpts) + }) + + t.Run("Expanded", func(t *testing.T) { + t.Parallel() + + factory := nodeResolvingFactory{ + testFactory: testFactory{ + name: "example", + }, + nodes: []driver.Node{ + { + Name: "node-amd64", + Endpoint: "tcp://amd64:1234", + Platforms: []string{"linux/amd64"}, + DriverOpts: map[string]string{"arch": "amd64"}, + }, + { + Name: "node-arm64", + Endpoint: "tcp://arm64:1234", + Platforms: []string{"linux/arm64"}, + DriverOpts: map[string]string{"arch": "arm64"}, + }, + }, + } + ng := &store.NodeGroup{ + Nodes: []store.Node{ + { + Name: "existing", + Endpoint: "tcp://existing:1234", + }, + }, + } + + require.NoError(t, updateNodeGroup(t.Context(), factory, ng, driver.Node{Endpoint: "org/builder"}, true, nil, "")) + require.Len(t, ng.Nodes, 3) + assert.Equal(t, "node-amd64", ng.Nodes[1].Name) + assert.Equal(t, "tcp://amd64:1234", ng.Nodes[1].Endpoint) + assert.Equal(t, "amd64", ng.Nodes[1].Platforms[0].Architecture) + assert.Equal(t, map[string]string{"arch": "amd64"}, ng.Nodes[1].DriverOpts) + assert.Equal(t, "node-arm64", ng.Nodes[2].Name) + assert.Equal(t, "tcp://arm64:1234", ng.Nodes[2].Endpoint) + assert.Equal(t, "arm64", ng.Nodes[2].Platforms[0].Architecture) + assert.Equal(t, map[string]string{"arch": "arm64"}, ng.Nodes[2].DriverOpts) + }) + + t.Run("ExpandedUpdatePreservesEndpoint", func(t *testing.T) { + t.Parallel() + + factory := nodeResolvingFactory{ + testFactory: testFactory{ + name: "example", + }, + nodes: []driver.Node{ + { + Name: "node", + Endpoint: "tcp://new:1234", + Platforms: []string{"linux/arm64"}, + EndpointSet: false, + }, + }, + } + ng := &store.NodeGroup{ + Nodes: []store.Node{ + { + Name: "node", + Endpoint: "tcp://existing:1234", + }, + }, + } + + require.NoError(t, updateNodeGroup(t.Context(), factory, ng, driver.Node{Endpoint: "org/builder"}, false, nil, "")) + require.Len(t, ng.Nodes, 1) + assert.Equal(t, "tcp://existing:1234", ng.Nodes[0].Endpoint) + assert.Equal(t, "arm64", ng.Nodes[0].Platforms[0].Architecture) + }) + + t.Run("ExpandedAppendRejectsNameCollision", func(t *testing.T) { + t.Parallel() + + factory := nodeResolvingFactory{ + testFactory: testFactory{ + name: "example", + }, + nodes: []driver.Node{ + { + Name: "node", + Endpoint: "tcp://new:1234", + Platforms: []string{"linux/arm64"}, + DriverOpts: map[string]string{"arch": "arm64"}, + }, + }, + } + ng := &store.NodeGroup{ + Nodes: []store.Node{ + { + Name: "node", + Endpoint: "tcp://existing:1234", + DriverOpts: map[string]string{"arch": "amd64"}, + }, + }, + } + + err := updateNodeGroup(t.Context(), factory, ng, driver.Node{Endpoint: "org/builder"}, true, nil, "") + require.EqualError(t, err, `node "node" already exists`) + require.Len(t, ng.Nodes, 1) + assert.Equal(t, "tcp://existing:1234", ng.Nodes[0].Endpoint) + assert.Equal(t, map[string]string{"arch": "amd64"}, ng.Nodes[0].DriverOpts) + }) + + t.Run("EmptyExpansion", func(t *testing.T) { + t.Parallel() + + factory := nodeResolvingFactory{ + testFactory: testFactory{ + name: "example", + }, + } + + err := updateNodeGroup(t.Context(), factory, &store.NodeGroup{}, driver.Node{}, false, nil, "") + require.EqualError(t, err, `driver "example" returned no nodes`) + }) +} + func TestCsvToMap(t *testing.T) { d := []string{ "\"tolerations=key=foo,value=bar;key=foo2,value=bar2\",replicas=1", diff --git a/driver/cloud/create.go b/driver/cloud/create.go index 563e8d761948..dbe2ed956ed5 100644 --- a/driver/cloud/create.go +++ b/driver/cloud/create.go @@ -11,13 +11,14 @@ import ( "strings" "time" + "github.com/docker/buildx/driver" "github.com/pkg/errors" ) const hubHostDefault = "https://hub.docker.com" -// CloudBuilder represents a builder instance in as returned by Hub Cloud Builds API. -type CloudBuilder struct { +// cloudBuilder represents a builder instance returned by the Hub Cloud Builds API. +type cloudBuilder struct { // Name of the builder e.g. linux-amd64. Name string `json:"name"` // Platform of the builder e.g. linux/amd64. @@ -25,10 +26,10 @@ type CloudBuilder struct { // Endpoint of the builder e.g. cloud://myorg/mybuilder_linux-amd64 Endpoint string `json:"endpoint"` // DataPlane contains data plane info for the builder - DataPlane CloudBuilderDataPlane `json:"data_plane"` + DataPlane cloudBuilderDataPlane `json:"data_plane"` } -type CloudBuilderDataPlane struct { +type cloudBuilderDataPlane struct { // Name is the logical name of the data plane e.g. us-east-1 Name string `json:"name"` // DisplayName is the name to display to users e.g. US East @@ -43,7 +44,7 @@ type CloudBuilderDataPlane struct { // Arch returns the architecture of the builder. // example: for platform linux/amd64, returns amd64. -func (b CloudBuilder) Arch() string { +func (b cloudBuilder) Arch() string { parts := strings.Split(b.Platform, "/") if len(parts) != 2 { return "" @@ -51,9 +52,9 @@ func (b CloudBuilder) Arch() string { return parts[1] } -// GetBuilderInstances calls the Hub Cloud Builds API to get +// resolveBuilderInstances calls the Hub Cloud Builds API to get // a list of builder instances under a builder group. -func GetBuilderInstances(ctx context.Context, group string, driverOpts map[string]string) ([]CloudBuilder, error) { +func resolveBuilderInstances(ctx context.Context, group string, driverOpts map[string]string) ([]cloudBuilder, error) { ctx, cancel := context.WithTimeoutCause(ctx, 30*time.Second, errors.WithStack(context.DeadlineExceeded)) defer cancel() registryHostname := hubRegistryEntry @@ -93,9 +94,9 @@ func GetBuilderInstances(ctx context.Context, group string, driverOpts map[strin return builders, nil } -// GetBuilderInstanceDriverOpts creates the instance specific driver opts +// getBuilderInstanceDriverOpts creates the instance specific driver opts // from the driver-opt args and the builder instance details -func GetBuilderInstanceDriverOpts(builderInstance CloudBuilder, driverOpts map[string]string) map[string]string { +func getBuilderInstanceDriverOpts(builderInstance cloudBuilder, driverOpts map[string]string) map[string]string { opts := maps.Clone(driverOpts) // If builder instance has a data plane endpoint and it has not been overridden by the driver opts then apply it if opts[optKeyInternalAddress] == "" { @@ -115,9 +116,31 @@ func GetBuilderInstanceDriverOpts(builderInstance CloudBuilder, driverOpts map[s return opts } +func (*factory) ResolveNodes(ctx context.Context, node driver.Node) ([]driver.Node, error) { + if node.Endpoint == "" { + return nil, errors.Errorf("no endpoint (builder) provided") + } + builders, err := resolveBuilderInstances(ctx, node.Endpoint, node.DriverOpts) + if err != nil { + return nil, errors.Wrap(err, "unknown builder") + } + + nodes := make([]driver.Node, len(builders)) + for i, builder := range builders { + nodes[i] = driver.Node{ + Name: builder.Name, + Endpoint: builder.Endpoint, + Platforms: []string{builder.Platform}, + EndpointSet: false, + DriverOpts: getBuilderInstanceDriverOpts(builder, node.DriverOpts), + } + } + return nodes, nil +} + // getBuilderInstances calls the Hub Cloud Builds API to get // a list of builder instances under a builder group. -func getBuilderInstances(ctx context.Context, builderGroup, hubHost, token string) ([]CloudBuilder, error) { +func getBuilderInstances(ctx context.Context, builderGroup, hubHost, token string) ([]cloudBuilder, error) { parts := strings.Split(builderGroup, "/") namespace := parts[0] group := parts[1] @@ -141,7 +164,7 @@ func getBuilderInstances(ctx context.Context, builderGroup, hubHost, token strin // ignoring pagination as the default page size of 10 should include all builders. var respPayload struct { - Results []CloudBuilder `json:"results"` + Results []cloudBuilder `json:"results"` } if err := json.NewDecoder(resp.Body).Decode(&respPayload); err != nil { return nil, errors.Wrap(err, "decode get instances response") diff --git a/driver/cloud/create_test.go b/driver/cloud/create_test.go index ecc958ad05a9..6dc6924471ab 100644 --- a/driver/cloud/create_test.go +++ b/driver/cloud/create_test.go @@ -3,16 +3,25 @@ package cloud import ( "testing" + "github.com/docker/buildx/driver" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestResolveNodesRequiresEndpoint(t *testing.T) { + t.Parallel() + + _, err := (&factory{}).ResolveNodes(t.Context(), driver.Node{}) + require.EqualError(t, err, "no endpoint (builder) provided") +} + func TestGetBuilderInstanceDriverOpts(t *testing.T) { t.Parallel() testCases := []struct { name string driverOpts map[string]string - builderInstance CloudBuilder + builderInstance cloudBuilder expected map[string]string }{ { @@ -22,8 +31,8 @@ func TestGetBuilderInstanceDriverOpts(t *testing.T) { }, { name: "DataPlaneOnly", - builderInstance: CloudBuilder{ - DataPlane: CloudBuilderDataPlane{ + builderInstance: cloudBuilder{ + DataPlane: cloudBuilderDataPlane{ ProxyEndpoint: "tcp://endpoint:443", RegistryEndpoint: "https://endpoint:443", HealthEndpoint: "https://endpoint:443/v2/", @@ -52,8 +61,8 @@ func TestGetBuilderInstanceDriverOpts(t *testing.T) { }, { name: "InternalAddressOverridesDataPlane", - builderInstance: CloudBuilder{ - DataPlane: CloudBuilderDataPlane{ + builderInstance: cloudBuilder{ + DataPlane: cloudBuilderDataPlane{ ProxyEndpoint: "tcp://endpoint:443", RegistryEndpoint: "https://endpoint:443", HealthEndpoint: "https://endpoint:443/v2/", @@ -70,8 +79,8 @@ func TestGetBuilderInstanceDriverOpts(t *testing.T) { }, { name: "PartialDataPlaneOverride", - builderInstance: CloudBuilder{ - DataPlane: CloudBuilderDataPlane{ + builderInstance: cloudBuilder{ + DataPlane: cloudBuilderDataPlane{ ProxyEndpoint: "tcp://endpoint:443", RegistryEndpoint: "https://endpoint:443", HealthEndpoint: "https://endpoint:443/v2/", @@ -94,7 +103,7 @@ func TestGetBuilderInstanceDriverOpts(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - result := GetBuilderInstanceDriverOpts(tc.builderInstance, tc.driverOpts) + result := getBuilderInstanceDriverOpts(tc.builderInstance, tc.driverOpts) assert.Equal(t, tc.expected, result) }) diff --git a/driver/cloud/factory.go b/driver/cloud/factory.go index 159f32d14411..a2db3eb78d08 100644 --- a/driver/cloud/factory.go +++ b/driver/cloud/factory.go @@ -72,7 +72,10 @@ type factory struct { once sync.Once } -var _ driver.DefaultBuilderNamer = (*factory)(nil) +var ( + _ driver.DefaultBuilderNamer = (*factory)(nil) + _ driver.NodeResolver = (*factory)(nil) +) func (*factory) Name() string { return DriverName diff --git a/driver/manager.go b/driver/manager.go index db7f299f1b5a..295c924d18e1 100644 --- a/driver/manager.go +++ b/driver/manager.go @@ -29,6 +29,18 @@ type DefaultBuilderNamer interface { DefaultBuilderName(ctx context.Context, endpoint string) (string, error) } +type Node struct { + Name string + Endpoint string + Platforms []string + EndpointSet bool + DriverOpts map[string]string +} + +type NodeResolver interface { + ResolveNodes(context.Context, Node) ([]Node, error) +} + type BuildkitConfig struct { // Entitlements []string // Rootless bool From de05d88f6053ae750e22ce5c2a22ebea8f131128 Mon Sep 17 00:00:00 2001 From: Alberto Garcia Hierro Date: Thu, 30 Jul 2026 17:54:06 +0200 Subject: [PATCH 05/21] build: let drivers prepare solve options Co-authored-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- build/build.go | 35 ++-- build/opt.go | 128 +++++------- build/output_test.go | 80 ++++++++ driver/cloud/driver.go | 183 +++++++++++------ driver/cloud/factory.go | 32 +-- driver/cloud/output_test.go | 393 ++++++++++++++++++++++++++++++++++++ driver/preparer.go | 27 +++ 7 files changed, 705 insertions(+), 173 deletions(-) create mode 100644 build/output_test.go create mode 100644 driver/cloud/output_test.go create mode 100644 driver/preparer.go diff --git a/build/build.go b/build/build.go index 6abb7d98a1c6..8aa75335c470 100644 --- a/build/build.go +++ b/build/build.go @@ -21,7 +21,6 @@ import ( noderesolver "github.com/docker/buildx/build/resolver" "github.com/docker/buildx/builder" "github.com/docker/buildx/driver" - "github.com/docker/buildx/driver/cloud" "github.com/docker/buildx/util/buildflags" "github.com/docker/buildx/util/confutil" "github.com/docker/buildx/util/desktop" @@ -262,32 +261,37 @@ func findNonMobyDriver(nodes []builder.Node) *driver.DriverHandle { return nil } -// warnOnNoOutput will check if the given nodes and options would result in an output -// and prints a warning if it would not. -func warnOnNoOutput(ctx context.Context, nodes []builder.Node, opts map[string]Options) { +func shouldWarnNoOutput(opt Options, reqs []*reqForNode) bool { + if opt.Linked || len(opt.Exports) > 0 { + return false + } + for _, req := range reqs { + if len(req.so.Exports) > 0 || req.so.EnableSessionExporter { + return false + } + } + return true +} + +// warnOnNoOutput checks if the prepared build requests would result in an +// output and prints a warning if they would not. +func warnOnNoOutput(nodes []builder.Node, opts map[string]Options, reqForNodes map[string][]*reqForNode) { // Return immediately if default load is explicitly disabled or a call // function is used. if noDefaultLoad() || !noCallFunc(opts) { return } - // Find the first non-moby driver and return if it either doesn't exist - // or if the driver has default load enabled. + // Find the first non-moby driver and return if it doesn't exist. noMobyDriver := findNonMobyDriver(nodes) - if noMobyDriver == nil || noMobyDriver.Features(ctx)[driver.DefaultLoad] { + if noMobyDriver == nil { return } // Produce a warning describing the targets affected. var noOutputTargets []string for name, opt := range opts { - if !opt.Linked && len(opt.Exports) == 0 { - // downstream toSolveOpt will force an image exporter if no exports are specified - // which, along with > 0 tags, will trigger a cloud pull in the cloud driver - implicitCloudPull := noMobyDriver.Factory().Name() == cloud.DriverName && len(opt.Tags) > 0 - if implicitCloudPull { - continue - } + if shouldWarnNoOutput(opt, reqForNodes[name]) { noOutputTargets = append(noOutputTargets, name) } } @@ -497,8 +501,6 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[ if err != nil { return nil, errors.Wrapf(err, "no valid drivers found") } - warnOnNoOutput(ctx, nodes, opts) - optPlatforms := make(map[string][]ocispecs.Platform, len(opts)) for k, opt := range opts { optPlatforms[k] = opt.Platforms @@ -516,6 +518,7 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opts map[ defer func() { release(err) }() + warnOnNoOutput(nodes, opts, reqForNodes) // validate that all links between targets use same drivers if err := validateTargetLinks(reqForNodes, drivers, opts); err != nil { diff --git a/build/opt.go b/build/opt.go index 32da70f8e152..47154f927830 100644 --- a/build/opt.go +++ b/build/opt.go @@ -23,7 +23,6 @@ import ( "github.com/distribution/reference" noderesolver "github.com/docker/buildx/build/resolver" "github.com/docker/buildx/driver" - "github.com/docker/buildx/driver/cloud" "github.com/docker/buildx/policy" "github.com/docker/buildx/util/buildflags" "github.com/docker/buildx/util/confutil" @@ -42,7 +41,6 @@ import ( gateway "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/session" - "github.com/moby/buildkit/session/exporter/exporterprovider" "github.com/moby/buildkit/session/secrets/secretsprovider" "github.com/moby/buildkit/session/sshforward/sshprovider" "github.com/moby/buildkit/session/upload/uploadprovider" @@ -374,6 +372,16 @@ func toSolveOpt(ctx context.Context, np *noderesolver.ResolvedNode, multiDriver } } + buildInfoAttrs, buildInfoAttrsSet := opt.BuildArgs["BUILDKIT_INLINE_BUILDINFO_ATTRS"] + var noDefaultOCIArtifact bool + if v, ok := os.LookupEnv(noDefaultOCIArtifactEnv); ok { + noDefaultOCIArtifact, err = strconv.ParseBool(v) + if err != nil { + return nil, nil, errors.Wrap(err, "invalid "+noDefaultOCIArtifactEnv) + } + } + noDefaultOCIArtifact = noDefaultOCIArtifact && supportAttestations + switch len(opt.Exports) { case 1: // valid @@ -406,16 +414,10 @@ func toSolveOpt(ctx context.Context, np *noderesolver.ResolvedNode, multiDriver } } } - // modify exports to enable cloud pull. do this before setting up exporters, since - // the cloud pull might add or remove exporters. - if node.Driver.Factory().Name() == cloud.DriverName { - cloudDriver := node.Driver.Driver.(*cloud.Driver) - opt.Exports = cloudDriver.CheckCloudPull(ctx, docker, opt.Exports, opt.Tags) - } - // fill in image exporter names from tags + var tags []string if len(opt.Tags) > 0 { - tags := make([]string, len(opt.Tags)) + tags = make([]string, len(opt.Tags)) for i, tag := range opt.Tags { ref, err := reference.Parse(tag) if err != nil { @@ -425,7 +427,7 @@ func toSolveOpt(ctx context.Context, np *noderesolver.ResolvedNode, multiDriver } for i, e := range opt.Exports { switch e.Type { - case "image", "oci", "docker", cloud.CloudPullExportType: + case "image", "oci", "docker": opt.Exports[i].Attrs["name"] = strings.Join(tags, ",") } } @@ -439,26 +441,41 @@ func toSolveOpt(ctx context.Context, np *noderesolver.ResolvedNode, multiDriver } } - // cacheonly is a fake exporter to opt out of default behaviors - exports := make([]client.ExportEntry, 0, len(opt.Exports)) - for _, e := range opt.Exports { - if e.Type != "cacheonly" { - exports = append(exports, e) + so.Exports = slices.Clone(opt.Exports) + for i := range so.Exports { + so.Exports[i].Attrs = maps.Clone(so.Exports[i].Attrs) + } + so.Session = slices.Clone(opt.Session) + + if preparer, ok := nodeDriver.Driver.(driver.BuildPreparer); ok { + prepareOpt := driver.PrepareBuildOptions{ + SolveOpt: &so, + Docker: docker, + Tags: tags, + Platforms: opt.Platforms, + MultiDriver: multiDriver, + NoOutput: opt.CallFunc != nil, + BuildInfoAttrs: buildInfoAttrs, + BuildInfoAttrsSet: buildInfoAttrsSet, + NoDefaultOCIArtifact: noDefaultOCIArtifact, + Progress: pw, + } + if err := preparer.PrepareBuild(ctx, &prepareOpt); err != nil { + return nil, nil, errors.Wrap(err, "preparing build") } } - opt.Exports = exports - var noDefaultOCIArtifact bool - if v, ok := os.LookupEnv(noDefaultOCIArtifactEnv); ok { - noDefaultOCIArtifact, err = strconv.ParseBool(v) - if err != nil { - return nil, nil, errors.Wrap(err, "invalid "+noDefaultOCIArtifactEnv) + // cacheonly is a fake exporter to opt out of default behaviors + exports := make([]client.ExportEntry, 0, len(so.Exports)) + for _, e := range so.Exports { + if e.Type != "cacheonly" { + exports = append(exports, e) } } + so.Exports = exports // set up exporters - cloudPullConfigured := false - for i, e := range opt.Exports { + for i, e := range so.Exports { if e.Type == "oci" && !nodeDriver.Features(ctx)[driver.OCIExporter] { return nil, nil, notSupported(driver.OCIExporter, nodeDriver, "https://docs.docker.com/go/build-exporters/") } @@ -467,7 +484,7 @@ func toSolveOpt(ctx context.Context, np *noderesolver.ResolvedNode, multiDriver if features[dockerutil.OCIImporter] && e.Output == nil { // rely on oci importer if available (which supports // multi-platform images), otherwise fall back to docker - opt.Exports[i].Type = "oci" + so.Exports[i].Type = "oci" } else if len(opt.Platforms) > 1 || len(attests) > 0 { if e.Output != nil { return nil, nil, errors.Errorf("docker exporter does not support exporting manifest lists, use the oci exporter instead") @@ -485,13 +502,13 @@ func toSolveOpt(ctx context.Context, np *noderesolver.ResolvedNode, multiDriver defers = append(defers, func(error) { cancel() }) - opt.Exports[i].Output = func(_ map[string]string) (io.WriteCloser, error) { + so.Exports[i].Output = func(_ map[string]string) (io.WriteCloser, error) { return w, nil } // if docker is using the containerd snapshotter, prefer to export the image digest // (rather than the image config digest). See https://github.com/moby/moby/issues/45458. if features[dockerutil.OCIImporter] { - opt.Exports[i].Attrs["prefer-image-digest"] = "true" + so.Exports[i].Attrs["prefer-image-digest"] = "true" } } } else if !nodeDriver.Features(ctx)[driver.DockerExporter] { @@ -499,10 +516,10 @@ func toSolveOpt(ctx context.Context, np *noderesolver.ResolvedNode, multiDriver } } if e.Type == "image" && nodeDriver.IsMobyDriver() { - opt.Exports[i].Type = "moby" + so.Exports[i].Type = "moby" // The containerd image store resolves images by manifest or index digest. if nodeDriver.Features(ctx)[driver.PreferImageDigest] { - opt.Exports[i].Attrs["prefer-image-digest"] = "true" + so.Exports[i].Attrs["prefer-image-digest"] = "true" } if e.Attrs["push"] != "" { if ok, _ := strconv.ParseBool(e.Attrs["push"]); ok { @@ -514,61 +531,20 @@ func toSolveOpt(ctx context.Context, np *noderesolver.ResolvedNode, multiDriver } if e.Type == "docker" || e.Type == "image" || e.Type == "oci" { // inline buildinfo attrs from build arg - if v, ok := opt.BuildArgs["BUILDKIT_INLINE_BUILDINFO_ATTRS"]; ok { - opt.Exports[i].Attrs["buildinfo-attrs"] = v - } - } - if e.Type == cloud.CloudPullExportType { - // cloud pull if node is cloud driver - opt.Exports[i].Type = "image" - if node.Driver.Factory().Name() == cloud.DriverName { - if multiDriver { - return nil, nil, errors.Errorf("cloud pull for multi-node builds currently not supported") - } - if len(opt.Platforms) > 1 { - return nil, nil, errors.Errorf("cloud pull for single node multi-platform builds currently not supported") - } - if !cloudPullConfigured { - cloudPullConfigured = true - cloudDriver := node.Driver.Driver.(*cloud.Driver) - targetPlatforms := slices.Clone(opt.Platforms) - tags := slices.Clone(opt.Tags) - dockerContext := e.Attrs["context"] - callback := exporterprovider.New(nil, exporterprovider.WithFinalizeCallback(func(ctx context.Context, exporterResponse map[string]string) error { - pw := progress.ResetTime(pw) - return progress.Wrap("cloud pull", pw.Write, func(l progress.SubLogger) error { - imgDescriptor := exporterResponse[exptypes.ExporterImageDescriptorKey] - platform := exporterResponse[exptypes.ExporterPlatformsKey] - // Fallback: if platform is not in buildkit response - // but --platform was specified, use it - if platform == "" && len(targetPlatforms) == 1 { - platform = platforms.Format(targetPlatforms[0]) - } - - if err := cloudDriver.CloudPull(ctx, imgDescriptor, platform, docker, dockerContext, tags, l); err != nil { - return errors.Wrap(err, "pulling image from cloud") - } - return nil - }) - })) - opt.Session = append(opt.Session, callback) - so.EnableSessionExporter = true - } + if buildInfoAttrsSet { + so.Exports[i].Attrs["buildinfo-attrs"] = buildInfoAttrs } } - if noDefaultOCIArtifact && supportAttestations { - switch opt.Exports[i].Type { + if noDefaultOCIArtifact { + switch so.Exports[i].Type { case client.ExporterImage, client.ExporterOCI, "moby": - if _, ok := opt.Exports[i].Attrs[string(exptypes.OptKeyOCIArtifact)]; !ok { - opt.Exports[i].Attrs[string(exptypes.OptKeyOCIArtifact)] = "false" + if _, ok := so.Exports[i].Attrs[string(exptypes.OptKeyOCIArtifact)]; !ok { + so.Exports[i].Attrs[string(exptypes.OptKeyOCIArtifact)] = "false" } } } } - so.Exports = opt.Exports - so.Session = slices.Clone(opt.Session) - for k, v := range opt.BuildArgs { so.FrontendAttrs["build-arg:"+k] = v } diff --git a/build/output_test.go b/build/output_test.go new file mode 100644 index 000000000000..8acd30a72726 --- /dev/null +++ b/build/output_test.go @@ -0,0 +1,80 @@ +package build + +import ( + "testing" + + "github.com/moby/buildkit/client" + "github.com/stretchr/testify/assert" +) + +func TestShouldWarnNoOutput(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + opt Options + reqs []*reqForNode + wantWarning bool + }{ + { + name: "NoRequests", + wantWarning: true, + }, + { + name: "NoOutput", + reqs: []*reqForNode{{ + so: &client.SolveOpt{}, + }}, + wantWarning: true, + }, + { + name: "Exporter", + reqs: []*reqForNode{{ + so: &client.SolveOpt{ + Exports: []client.ExportEntry{{Type: "image"}}, + }, + }}, + }, + { + name: "SessionExporter", + reqs: []*reqForNode{{ + so: &client.SolveOpt{ + EnableSessionExporter: true, + }, + }}, + }, + { + name: "OutputOnOneNode", + reqs: []*reqForNode{ + { + so: &client.SolveOpt{}, + }, + { + so: &client.SolveOpt{ + Exports: []client.ExportEntry{{Type: "image"}}, + }, + }, + }, + }, + { + name: "Linked", + opt: Options{ + Linked: true, + }, + }, + { + name: "ExplicitCacheOnly", + opt: Options{ + Exports: []client.ExportEntry{{Type: "cacheonly"}}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.wantWarning, shouldWarnNoOutput(tt.opt, tt.reqs)) + }) + } +} diff --git a/driver/cloud/driver.go b/driver/cloud/driver.go index 34d86001285a..95580b9e45df 100644 --- a/driver/cloud/driver.go +++ b/driver/cloud/driver.go @@ -13,24 +13,27 @@ import ( "net" "net/http" "os" + "slices" "strconv" "strings" "time" + "github.com/Masterminds/semver/v3" "github.com/containerd/platforms" "github.com/docker/buildx/driver" "github.com/docker/buildx/util/dockerutil" "github.com/docker/buildx/util/progress" + "github.com/moby/buildkit/client" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + sessionexporter "github.com/moby/buildkit/session/exporter" + "github.com/moby/buildkit/session/exporter/exporterprovider" "github.com/moby/moby/api/types/jsonstream" "github.com/moby/moby/api/types/registry" "github.com/moby/moby/api/types/system" dockerclient "github.com/moby/moby/client" - "github.com/sirupsen/logrus" - - "github.com/Masterminds/semver/v3" - "github.com/moby/buildkit/client" specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "google.golang.org/grpc" grpcgzip "google.golang.org/grpc/encoding/gzip" "google.golang.org/grpc/keepalive" @@ -58,8 +61,10 @@ type Driver struct { headers []string } -// make sure cloud driver implements Driver interface -var _ driver.Driver = &Driver{} +var _ interface { + driver.Driver + driver.BuildPreparer +} = (*Driver)(nil) // tlsOpts allows overriding the default TLS configuration for communicating // with the registry. This should only be used for development and testing. @@ -192,9 +197,7 @@ func (d *Driver) HostGatewayIP(ctx context.Context) (net.IP, error) { return nil, errors.New("host-gateway is not supported by the cloud driver") } -// CloudPull pulls an image from the cloud registry. -// It should be called right after Solve returns. -func (d *Driver) CloudPull(ctx context.Context, imageDescriptor string, platform string, duc *dockerutil.Client, dockerContext string, tags []string, l progress.SubLogger) error { +func (d *Driver) cloudPull(ctx context.Context, imageDescriptor string, platform string, duc *dockerutil.Client, dockerContext string, tags []string, l progress.SubLogger) error { dc, err := duc.API(dockerContext) if err != nil { return err @@ -275,73 +278,133 @@ func (d *Driver) CloudPull(ctx context.Context, imageDescriptor string, platform return nil } -// CheckCloudPull checks if cloud pull should be enabled and returns the updated exports. -// Enables cloud pull if -// - cloud driver is used -// - daemon supports RegistryToken in ImagePullOptions -// - docker exporter or image exporter (without push) is used -// - at least one tag is set -func (d *Driver) CheckCloudPull(ctx context.Context, docker *dockerutil.Client, exports []client.ExportEntry, tags []string) []client.ExportEntry { - if len(tags) == 0 { - return exports - } - - dockerContext := "" - hasDockerContext := false - hasCloudPull := len(exports) == 0 - for _, e := range exports { - pushAttr, _ := strconv.ParseBool(e.Attrs["push"]) - if e.Type != "docker" && (e.Type != "image" || pushAttr) { - continue - } - if e.Output != nil || e.OutputDir != "" { - continue - } - hasCloudPull = true - context := e.Attrs["context"] - if !hasDockerContext { - dockerContext = context - hasDockerContext = true - } else if dockerContext != context { - return exports - } +func (d *Driver) PrepareBuild(ctx context.Context, opt *driver.PrepareBuildOptions) error { + if opt.NoOutput { + return nil + } + if len(opt.Tags) == 0 { + return nil } - if !hasCloudPull { - return exports + + exports, requests, dockerContext := resolveCloudPullExporters(opt.SolveOpt.Exports, opt.Tags, opt.BuildInfoAttrs, opt.BuildInfoAttrsSet, opt.NoDefaultOCIArtifact) + if len(requests) == 0 { + return nil + } + + if opt.MultiDriver || len(opt.Platforms) > 1 { + return nil } - supportsRegistryTokenPull, err := daemonSupportsRegistryTokenPull(ctx, docker, dockerContext) + supportsRegistryTokenPull, err := daemonSupportsRegistryTokenPull(ctx, opt.Docker, dockerContext) if err != nil { logrus.Warnf("failed to detect registry pull token support in daemon: %v", err) - return exports + return nil } - if !supportsRegistryTokenPull { - return exports + return nil } - if len(exports) == 0 { - attrs := maps.Clone(CouldPullExportRequiredAttributes) - return []client.ExportEntry{ - { - Type: CloudPullExportType, - Attrs: attrs, - }, + opt.SolveOpt.Exports = exports + opt.SolveOpt.Session = append(opt.SolveOpt.Session, d.newCloudPullSessionExporter(*opt, requests, dockerContext)) + opt.SolveOpt.EnableSessionExporter = true + return nil +} + +func (d *Driver) newCloudPullSessionExporter(opt driver.PrepareBuildOptions, requests []*sessionexporter.ExporterRequest, dockerContext string) *exporterprovider.Exporter { + targetPlatform := d.cloudPullPlatform(opt) + tags := slices.Clone(opt.Tags) + return exporterprovider.New(func(context.Context, map[string][]byte, []string) ([]*sessionexporter.ExporterRequest, error) { + return requests, nil + }, exporterprovider.WithFinalizeCallback(func(ctx context.Context, exporterResponse map[string]string) error { + imgDescriptor := exporterResponse[exptypes.ExporterImageDescriptorKey] + if imgDescriptor == "" { + return errors.Errorf("missing exporter response %q", exptypes.ExporterImageDescriptorKey) } + pw := progress.ResetTime(opt.Progress) + return progress.Wrap("cloud pull", pw.Write, func(l progress.SubLogger) error { + if err := d.cloudPull(ctx, imgDescriptor, targetPlatform, opt.Docker, dockerContext, tags, l); err != nil { + return errors.Wrap(err, "pulling image from cloud") + } + return nil + }) + })) +} + +func (d *Driver) cloudPullPlatform(opt driver.PrepareBuildOptions) string { + targetPlatforms := opt.Platforms + if len(targetPlatforms) == 0 { + targetPlatforms = d.Config().Platforms } + if len(targetPlatforms) != 1 { + return "" + } + return platforms.Format(targetPlatforms[0]) +} - for i, e := range exports { - pushAttr, _ := strconv.ParseBool(e.Attrs["push"]) - if e.Output != nil || e.OutputDir != "" { - continue +func resolveCloudPullExporters(exports []client.ExportEntry, tags []string, buildInfoAttrs string, buildInfoAttrsSet bool, noDefaultOCIArtifact bool) ([]client.ExportEntry, []*sessionexporter.ExporterRequest, string) { + exporterAttrs := func(attrs map[string]string) map[string]string { + attrs = maps.Clone(attrs) + if attrs == nil { + attrs = map[string]string{} } - if e.Type == "docker" || (e.Type == "image" && !pushAttr) { - exports[i].Type = CloudPullExportType - maps.Copy(exports[i].Attrs, CouldPullExportRequiredAttributes) + attrs["attestation-inline"] = "false" + if attrs["name"] == "" { + attrs["name"] = strings.Join(tags, ",") } + if buildInfoAttrsSet { + attrs["buildinfo-attrs"] = buildInfoAttrs + } + if noDefaultOCIArtifact { + if _, ok := attrs[string(exptypes.OptKeyOCIArtifact)]; !ok { + attrs[string(exptypes.OptKeyOCIArtifact)] = "false" + } + } + return attrs } - return exports + if len(exports) == 0 { + return nil, []*sessionexporter.ExporterRequest{{ + Type: "image", + Attrs: exporterAttrs(nil), + }}, "" + } + + var remaining []client.ExportEntry + var requests []*sessionexporter.ExporterRequest + var dockerContext string + var hasDockerContext bool + for _, export := range exports { + push, _ := strconv.ParseBool(export.Attrs["push"]) + if export.Type != "docker" && (export.Type != "image" || push) { + remaining = append(remaining, export) + continue + } + if export.Output != nil || export.OutputDir != "" { + remaining = append(remaining, export) + continue + } + dctx := export.Attrs["context"] + if !hasDockerContext { + dockerContext = dctx + hasDockerContext = true + } else if dockerContext != dctx { + return exports, nil, "" + } + request := &sessionexporter.ExporterRequest{ + Type: "image", + Attrs: exporterAttrs(export.Attrs), + } + if len(requests) > 0 { + // FinalizeExport receives one response map, so cloud pull only + // supports one distinct converted exporter request. + if requests[0].Type == request.Type && maps.Equal(requests[0].Attrs, request.Attrs) { + continue + } + return exports, nil, "" + } + requests = append(requests, request) + } + return remaining, requests, dockerContext } // tlsConfig returns a TLS configuration for communicating with the registry, diff --git a/driver/cloud/factory.go b/driver/cloud/factory.go index a2db3eb78d08..0e5b43c4f1c5 100644 --- a/driver/cloud/factory.go +++ b/driver/cloud/factory.go @@ -21,14 +21,11 @@ import ( const EndpointPrefix = "cloud://" const ( - DriverName = "cloud" - // CloudPullExportType is the type of export used for enabling - // cloud pulls. - CloudPullExportType = "cloud" - AuthHost = "https://auth.docker.io" - ProxyAddress = "tcp://build-cloud.docker.com:443" - RegistryAddress = "build-cloud.docker.com:443" - HealthAddress = "https://build-cloud.docker.com:443/v2/" + DriverName = "cloud" + AuthHost = "https://auth.docker.io" + ProxyAddress = "tcp://build-cloud.docker.com:443" + RegistryAddress = "build-cloud.docker.com:443" + HealthAddress = "https://build-cloud.docker.com:443/v2/" // We want a low priority for the cloud driver for now. lowPriority = 1000 @@ -55,28 +52,21 @@ const ( optKeyInternalHealthAddress = "internal.cloud.health.address" ) -var ( - // CouldPullExportRequiredAttributes is the list of required attributes for the Export entry - // when doing a cloud pull. - CouldPullExportRequiredAttributes = map[string]string{ - "attestation-inline": "false", - } -) - func init() { driver.Register(&factory{}) } +var _ interface { + driver.Factory + driver.DefaultBuilderNamer + driver.NodeResolver +} = (*factory)(nil) + type factory struct { headers []string once sync.Once } -var ( - _ driver.DefaultBuilderNamer = (*factory)(nil) - _ driver.NodeResolver = (*factory)(nil) -) - func (*factory) Name() string { return DriverName } diff --git a/driver/cloud/output_test.go b/driver/cloud/output_test.go new file mode 100644 index 000000000000..aafde43460d2 --- /dev/null +++ b/driver/cloud/output_test.go @@ -0,0 +1,393 @@ +package cloud + +import ( + "io" + "testing" + + "github.com/containerd/platforms" + "github.com/docker/buildx/driver" + "github.com/moby/buildkit/client" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + sessionexporter "github.com/moby/buildkit/session/exporter" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveCloudPullExporters(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + exports []client.ExportEntry + buildInfoAttrs string + buildInfoAttrsSet bool + noDefaultOCIArtifact bool + wantExports []client.ExportEntry + wantRequests []*sessionexporter.ExporterRequest + wantContext string + }{ + { + name: "Implicit", + wantRequests: []*sessionexporter.ExporterRequest{{ + Type: "image", + Attrs: map[string]string{ + "attestation-inline": "false", + "name": "example.com/image:latest", + }, + }}, + }, + { + name: "Docker", + exports: []client.ExportEntry{{ + Type: "docker", + Attrs: map[string]string{"name": "example.com/image:latest", "context": "default"}, + }}, + wantRequests: []*sessionexporter.ExporterRequest{{ + Type: "image", + Attrs: map[string]string{ + "attestation-inline": "false", + "context": "default", + "name": "example.com/image:latest", + }, + }}, + wantContext: "default", + }, + { + name: "Image", + exports: []client.ExportEntry{{ + Type: "image", + }}, + wantRequests: []*sessionexporter.ExporterRequest{{ + Type: "image", + Attrs: map[string]string{ + "attestation-inline": "false", + "name": "example.com/image:latest", + }, + }}, + }, + { + name: "Push", + exports: []client.ExportEntry{{ + Type: "image", + Attrs: map[string]string{"push": "true"}, + }}, + wantExports: []client.ExportEntry{{ + Type: "image", + Attrs: map[string]string{"push": "true"}, + }}, + }, + { + name: "OCI", + exports: []client.ExportEntry{{ + Type: "oci", + }}, + wantExports: []client.ExportEntry{{ + Type: "oci", + }}, + }, + { + name: "CacheOnly", + exports: []client.ExportEntry{{ + Type: "cacheonly", + }}, + wantExports: []client.ExportEntry{{ + Type: "cacheonly", + }}, + }, + { + name: "Mixed", + exports: []client.ExportEntry{ + {Type: "docker", Attrs: map[string]string{"compression": "zstd"}}, + {Type: "image", Attrs: map[string]string{"push": "true"}}, + {Type: "oci"}, + }, + wantExports: []client.ExportEntry{ + {Type: "image", Attrs: map[string]string{"push": "true"}}, + {Type: "oci"}, + }, + wantRequests: []*sessionexporter.ExporterRequest{{ + Type: "image", + Attrs: map[string]string{ + "attestation-inline": "false", + "compression": "zstd", + "name": "example.com/image:latest", + }, + }}, + }, + { + name: "GenericAttrs", + exports: []client.ExportEntry{{ + Type: "docker", + Attrs: map[string]string{"compression": "zstd"}, + }}, + buildInfoAttrs: "context,source", + buildInfoAttrsSet: true, + noDefaultOCIArtifact: true, + wantRequests: []*sessionexporter.ExporterRequest{{ + Type: "image", + Attrs: map[string]string{ + "attestation-inline": "false", + "buildinfo-attrs": "context,source", + "compression": "zstd", + "name": "example.com/image:latest", + string(exptypes.OptKeyOCIArtifact): "false", + }, + }}, + }, + { + name: "ExplicitOCIArtifact", + exports: []client.ExportEntry{{ + Type: "image", + Attrs: map[string]string{ + string(exptypes.OptKeyOCIArtifact): "true", + }, + }}, + noDefaultOCIArtifact: true, + wantRequests: []*sessionexporter.ExporterRequest{{ + Type: "image", + Attrs: map[string]string{ + "attestation-inline": "false", + "name": "example.com/image:latest", + string(exptypes.OptKeyOCIArtifact): "true", + }, + }}, + }, + { + name: "MultipleCloudPullRequests", + exports: []client.ExportEntry{ + {Type: "docker", Attrs: map[string]string{"compression": "zstd"}}, + {Type: "image", Attrs: map[string]string{"oci-mediatypes": "true"}}, + }, + wantExports: []client.ExportEntry{ + {Type: "docker", Attrs: map[string]string{"compression": "zstd"}}, + {Type: "image", Attrs: map[string]string{"oci-mediatypes": "true"}}, + }, + }, + { + name: "MultipleMatchingContexts", + exports: []client.ExportEntry{ + {Type: "docker", Attrs: map[string]string{"context": "staging"}}, + {Type: "docker", Attrs: map[string]string{"context": "staging"}}, + }, + wantRequests: []*sessionexporter.ExporterRequest{{ + Type: "image", + Attrs: map[string]string{ + "attestation-inline": "false", + "context": "staging", + "name": "example.com/image:latest", + }, + }}, + wantContext: "staging", + }, + { + name: "MixedCurrentAndNamedContext", + exports: []client.ExportEntry{ + {Type: "docker"}, + {Type: "docker", Attrs: map[string]string{"context": "staging"}}, + }, + wantExports: []client.ExportEntry{ + {Type: "docker"}, + {Type: "docker", Attrs: map[string]string{"context": "staging"}}, + }, + }, + { + name: "ConflictingContexts", + exports: []client.ExportEntry{ + {Type: "docker", Attrs: map[string]string{"context": "staging"}}, + {Type: "docker", Attrs: map[string]string{"context": "prod"}}, + }, + wantExports: []client.ExportEntry{ + {Type: "docker", Attrs: map[string]string{"context": "staging"}}, + {Type: "docker", Attrs: map[string]string{"context": "prod"}}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + exports, requests, dockerContext := resolveCloudPullExporters(tt.exports, []string{"example.com/image:latest"}, tt.buildInfoAttrs, tt.buildInfoAttrsSet, tt.noDefaultOCIArtifact) + assert.Equal(t, tt.wantExports, exports) + assert.Equal(t, tt.wantRequests, requests) + assert.Equal(t, tt.wantContext, dockerContext) + }) + } +} + +func TestResolveCloudPullExportersClonesAttrs(t *testing.T) { + t.Parallel() + + exports := []client.ExportEntry{{ + Type: "docker", + Attrs: map[string]string{"compression": "zstd"}, + }} + _, requests, _ := resolveCloudPullExporters(exports, []string{"example.com/image:latest"}, "", false, false) + require.Len(t, requests, 1) + requests[0].Attrs["new"] = "value" + assert.NotContains(t, exports[0].Attrs, "attestation-inline") + assert.NotContains(t, exports[0].Attrs, "new") +} + +func TestResolveCloudPullExportersKeepsExplicitDockerOutput(t *testing.T) { + t.Parallel() + + output := func(map[string]string) (io.WriteCloser, error) { + return nil, nil + } + exports := []client.ExportEntry{ + {Type: "docker", Output: output}, + {Type: "docker", OutputDir: "out"}, + } + + gotExports, requests, dockerContext := resolveCloudPullExporters(exports, []string{"example.com/image:latest"}, "", false, false) + + require.Empty(t, requests) + assert.Empty(t, dockerContext) + assert.Len(t, gotExports, 2) + assert.NotNil(t, gotExports[0].Output) + assert.Equal(t, "out", gotExports[1].OutputDir) +} + +func TestPrepareBuildWithoutTags(t *testing.T) { + t.Parallel() + + exports := []client.ExportEntry{{Type: "docker"}} + solveOpt := client.SolveOpt{ + Exports: exports, + } + opt := driver.PrepareBuildOptions{ + SolveOpt: &solveOpt, + } + require.NoError(t, (&Driver{}).PrepareBuild(t.Context(), &opt)) + assert.Equal(t, exports, solveOpt.Exports) + assert.Empty(t, solveOpt.Session) + assert.False(t, solveOpt.EnableSessionExporter) +} + +func TestPrepareBuildNoOutput(t *testing.T) { + t.Parallel() + + solveOpt := client.SolveOpt{} + opt := driver.PrepareBuildOptions{ + SolveOpt: &solveOpt, + Tags: []string{"example.com/image:latest"}, + NoOutput: true, + } + + require.NoError(t, (&Driver{}).PrepareBuild(t.Context(), &opt)) + assert.Empty(t, solveOpt.Exports) + assert.Empty(t, solveOpt.Session) + assert.False(t, solveOpt.EnableSessionExporter) +} + +func TestPrepareBuildUnsupportedCloudPullFallsBack(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + multiDriver bool + platforms []ocispecs.Platform + }{ + { + name: "MultiDriver", + multiDriver: true, + }, + { + name: "MultiPlatform", + platforms: []ocispecs.Platform{ + {OS: "linux", Architecture: "amd64"}, + {OS: "linux", Architecture: "arm64"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + exports := []client.ExportEntry{{Type: "docker"}} + solveOpt := client.SolveOpt{ + Exports: exports, + } + opt := driver.PrepareBuildOptions{ + SolveOpt: &solveOpt, + Tags: []string{"example.com/image:latest"}, + Platforms: tt.platforms, + MultiDriver: tt.multiDriver, + } + + require.NoError(t, (&Driver{}).PrepareBuild(t.Context(), &opt)) + assert.Equal(t, exports, solveOpt.Exports) + assert.Empty(t, solveOpt.Session) + assert.False(t, solveOpt.EnableSessionExporter) + }) + } +} + +func TestCloudPullSessionExporter(t *testing.T) { + t.Parallel() + + requests := []*sessionexporter.ExporterRequest{{ + Type: "image", + Attrs: map[string]string{"name": "example.com/image:latest"}, + }} + provider := (&Driver{}).newCloudPullSessionExporter(driver.PrepareBuildOptions{}, requests, "") + + response, err := provider.FindExporters(t.Context(), &sessionexporter.FindExportersRequest{}) + require.NoError(t, err) + assert.Equal(t, requests, response.Exporters) +} + +func TestCloudPullPlatform(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + nodePlatforms []ocispecs.Platform + optPlatforms []ocispecs.Platform + want string + }{ + { + name: "RequestedPlatform", + nodePlatforms: []ocispecs.Platform{ + platforms.MustParse("linux/arm64"), + }, + optPlatforms: []ocispecs.Platform{ + platforms.MustParse("linux/amd64"), + }, + want: "linux/amd64", + }, + { + name: "SingleNodePlatform", + nodePlatforms: []ocispecs.Platform{ + platforms.MustParse("linux/arm64"), + }, + want: "linux/arm64", + }, + { + name: "MultipleNodePlatforms", + nodePlatforms: []ocispecs.Platform{ + platforms.MustParse("linux/amd64"), + platforms.MustParse("linux/arm64"), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + d := &Driver{ + InitConfig: driver.InitConfig{ + Platforms: tt.nodePlatforms, + }, + } + got := d.cloudPullPlatform(driver.PrepareBuildOptions{ + Platforms: tt.optPlatforms, + }) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/driver/preparer.go b/driver/preparer.go new file mode 100644 index 000000000000..e39022e3e812 --- /dev/null +++ b/driver/preparer.go @@ -0,0 +1,27 @@ +package driver + +import ( + "context" + + "github.com/docker/buildx/util/dockerutil" + "github.com/docker/buildx/util/progress" + "github.com/moby/buildkit/client" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" +) + +type PrepareBuildOptions struct { + SolveOpt *client.SolveOpt + Docker *dockerutil.Client + Tags []string + Platforms []ocispecs.Platform + MultiDriver bool + NoOutput bool // prevents drivers from adding output for metadata-only requests + BuildInfoAttrs string + BuildInfoAttrsSet bool + NoDefaultOCIArtifact bool + Progress progress.Writer +} + +type BuildPreparer interface { + PrepareBuild(ctx context.Context, opts *PrepareBuildOptions) error +} From 85eaeb576409bdbcfc6edeaf520b49a42e046243 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:39:50 +0200 Subject: [PATCH 06/21] cloud: validate builder names before resolving nodes Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- driver/cloud/create.go | 43 ++++++++++++++--- driver/cloud/create_test.go | 94 ++++++++++++++++++++++++++++++++++++ driver/cloud/factory.go | 14 ++++-- driver/cloud/factory_test.go | 7 +++ 4 files changed, 148 insertions(+), 10 deletions(-) diff --git a/driver/cloud/create.go b/driver/cloud/create.go index dbe2ed956ed5..f84294a10489 100644 --- a/driver/cloud/create.go +++ b/driver/cloud/create.go @@ -6,6 +6,7 @@ import ( "fmt" "maps" "net/http" + "net/url" "runtime" "sort" "strings" @@ -52,9 +53,34 @@ func (b cloudBuilder) Arch() string { return parts[1] } +func parseBuilderName(builder string) (account, name string, _ error) { + builder = strings.TrimPrefix(builder, EndpointPrefix) + account, name, ok := strings.Cut(builder, "/") + switch { + case !ok, + strings.Contains(name, "/"), + account == "", account == ".", account == "..", + name == "", name == ".", name == "..": + return "", "", errors.Errorf("builder should be in the format: /") + } + return account, name, nil +} + +func normalizeBuilderName(builder string) (string, error) { + account, name, err := parseBuilderName(builder) + if err != nil { + return "", err + } + return account + "/" + name, nil +} + // resolveBuilderInstances calls the Hub Cloud Builds API to get // a list of builder instances under a builder group. func resolveBuilderInstances(ctx context.Context, group string, driverOpts map[string]string) ([]cloudBuilder, error) { + group, err := normalizeBuilderName(group) + if err != nil { + return nil, err + } ctx, cancel := context.WithTimeoutCause(ctx, 30*time.Second, errors.WithStack(context.DeadlineExceeded)) defer cancel() registryHostname := hubRegistryEntry @@ -120,7 +146,11 @@ func (*factory) ResolveNodes(ctx context.Context, node driver.Node) ([]driver.No if node.Endpoint == "" { return nil, errors.Errorf("no endpoint (builder) provided") } - builders, err := resolveBuilderInstances(ctx, node.Endpoint, node.DriverOpts) + group, err := normalizeBuilderName(node.Endpoint) + if err != nil { + return nil, err + } + builders, err := resolveBuilderInstances(ctx, group, node.DriverOpts) if err != nil { return nil, errors.Wrap(err, "unknown builder") } @@ -141,12 +171,13 @@ func (*factory) ResolveNodes(ctx context.Context, node driver.Node) ([]driver.No // getBuilderInstances calls the Hub Cloud Builds API to get // a list of builder instances under a builder group. func getBuilderInstances(ctx context.Context, builderGroup, hubHost, token string) ([]cloudBuilder, error) { - parts := strings.Split(builderGroup, "/") - namespace := parts[0] - group := parts[1] - url := fmt.Sprintf("%s/v2/cloud-builds/accounts/%s/builder-groups/%s/instances", hubHost, namespace, group) + namespace, group, err := parseBuilderName(builderGroup) + if err != nil { + return nil, err + } + endpointURL := fmt.Sprintf("%s/v2/cloud-builds/accounts/%s/builder-groups/%s/instances", hubHost, url.PathEscape(namespace), url.PathEscape(group)) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpointURL, nil) if err != nil { return nil, errors.Wrap(err, "create request") } diff --git a/driver/cloud/create_test.go b/driver/cloud/create_test.go index 6dc6924471ab..1eb8e7c53c6d 100644 --- a/driver/cloud/create_test.go +++ b/driver/cloud/create_test.go @@ -8,6 +8,86 @@ import ( "github.com/stretchr/testify/require" ) +func TestParseBuilderName(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + builder string + wantAccount string + wantName string + wantErr bool + }{ + { + name: "Group", + builder: "org/builder", + wantAccount: "org", + wantName: "builder", + }, + { + name: "Endpoint", + builder: "cloud://org/builder", + wantAccount: "org", + wantName: "builder", + }, + { + name: "MissingName", + builder: "org", + wantErr: true, + }, + { + name: "EmptyAccount", + builder: "/builder", + wantErr: true, + }, + { + name: "EmptyName", + builder: "org/", + wantErr: true, + }, + { + name: "TooManyParts", + builder: "org/builder/extra", + wantErr: true, + }, + { + name: "DotAccount", + builder: "./builder", + wantErr: true, + }, + { + name: "DotDotAccount", + builder: "../builder", + wantErr: true, + }, + { + name: "DotName", + builder: "org/.", + wantErr: true, + }, + { + name: "DotDotName", + builder: "org/..", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + account, name, err := parseBuilderName(tt.builder) + if tt.wantErr { + require.EqualError(t, err, "builder should be in the format: /") + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantAccount, account) + assert.Equal(t, tt.wantName, name) + }) + } +} + func TestResolveNodesRequiresEndpoint(t *testing.T) { t.Parallel() @@ -15,6 +95,20 @@ func TestResolveNodesRequiresEndpoint(t *testing.T) { require.EqualError(t, err, "no endpoint (builder) provided") } +func TestResolveNodesRejectsMalformedEndpoint(t *testing.T) { + t.Parallel() + + _, err := (&factory{}).ResolveNodes(t.Context(), driver.Node{Endpoint: "org"}) + require.EqualError(t, err, "builder should be in the format: /") +} + +func TestGetBuilderInstancesRejectsMalformedBuilder(t *testing.T) { + t.Parallel() + + _, err := getBuilderInstances(t.Context(), "org", "http://example.com", "token") + require.EqualError(t, err, "builder should be in the format: /") +} + func TestGetBuilderInstanceDriverOpts(t *testing.T) { t.Parallel() diff --git a/driver/cloud/factory.go b/driver/cloud/factory.go index 0e5b43c4f1c5..aca91f59db25 100644 --- a/driver/cloud/factory.go +++ b/driver/cloud/factory.go @@ -76,7 +76,11 @@ func (*factory) Usage() string { } func (*factory) DefaultBuilderName(_ context.Context, endpoint string) (string, error) { - return DriverName + "-" + strings.ReplaceAll(strings.ToLower(endpoint), "/", "-"), nil + builder, err := normalizeBuilderName(endpoint) + if err != nil { + return "", err + } + return DriverName + "-" + strings.ReplaceAll(strings.ToLower(builder), "/", "-"), nil } func (*factory) Priority(ctx context.Context, endpoint string, api dockerclient.APIClient, _ map[string][]string) int { @@ -97,7 +101,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver proxyAddress: ProxyAddress, registryAddress: RegistryAddress, healthAddress: HealthAddress, - builder: strings.TrimPrefix(cfg.EndpointAddr, EndpointPrefix), + builder: cfg.EndpointAddr, headers: f.headers, } @@ -170,9 +174,11 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver return nil, errors.Errorf("builder name missing") } - if len(strings.Split(d.builder, "/")) != 2 { - return nil, errors.Errorf("builder should be in the format: /") + builder, err := normalizeBuilderName(d.builder) + if err != nil { + return nil, err } + d.builder = builder d.tokenSource = newTokenSource(func(ctx context.Context) (string, error) { return login(ctx, authEntry, authHost, d.builder) diff --git a/driver/cloud/factory_test.go b/driver/cloud/factory_test.go index a2995186c426..9d62d58cd9a4 100644 --- a/driver/cloud/factory_test.go +++ b/driver/cloud/factory_test.go @@ -20,6 +20,13 @@ func TestDefaultBuilderName(t *testing.T) { name, err := namer.DefaultBuilderName(t.Context(), "Org/Builder") require.NoError(t, err) assert.Equal(t, "cloud-org-builder", name) + + name, err = namer.DefaultBuilderName(t.Context(), "cloud://Org/Builder") + require.NoError(t, err) + assert.Equal(t, "cloud-org-builder", name) + + _, err = namer.DefaultBuilderName(t.Context(), "Org") + require.EqualError(t, err, "builder should be in the format: /") } func TestTokenRefresh(t *testing.T) { From 0b337b70b5ce558cd03d23c387640bc7c0c2c637 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:00:28 +0200 Subject: [PATCH 07/21] cloud: harden builder instance requests Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- driver/cloud/create.go | 13 ++++++---- driver/cloud/create_test.go | 47 +++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/driver/cloud/create.go b/driver/cloud/create.go index f84294a10489..af31807de469 100644 --- a/driver/cloud/create.go +++ b/driver/cloud/create.go @@ -3,7 +3,7 @@ package cloud import ( "context" "encoding/json" - "fmt" + "io" "maps" "net/http" "net/url" @@ -175,7 +175,10 @@ func getBuilderInstances(ctx context.Context, builderGroup, hubHost, token strin if err != nil { return nil, err } - endpointURL := fmt.Sprintf("%s/v2/cloud-builds/accounts/%s/builder-groups/%s/instances", hubHost, url.PathEscape(namespace), url.PathEscape(group)) + endpointURL, err := url.JoinPath(hubHost, "v2", "cloud-builds", "accounts", namespace, "builder-groups", group, "instances") + if err != nil { + return nil, errors.Wrap(err, "create get instances URL") + } req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpointURL, nil) if err != nil { @@ -183,14 +186,16 @@ func getBuilderInstances(ctx context.Context, builderGroup, hubHost, token strin } req.Header.Set("Authorization", "Bearer "+token) - resp, err := http.DefaultClient.Do(req) + resp, err := newClient(nil).Do(req) if err != nil { return nil, errors.Wrap(err, "request failed") } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, errors.New("response status code: " + resp.Status) + body, _ := io.ReadAll(io.LimitReader(resp.Body, 255)) + return nil, errors.Errorf("failed to get builders, got: %s. Trace ID: %s. Response body: %s", resp.Status, + resp.Header.Get("x-trace-id"), string(body)) } // ignoring pagination as the default page size of 10 should include all builders. diff --git a/driver/cloud/create_test.go b/driver/cloud/create_test.go index 1eb8e7c53c6d..7973235fcebd 100644 --- a/driver/cloud/create_test.go +++ b/driver/cloud/create_test.go @@ -1,6 +1,9 @@ package cloud import ( + "encoding/json" + "net/http" + "net/http/httptest" "testing" "github.com/docker/buildx/driver" @@ -109,6 +112,50 @@ func TestGetBuilderInstancesRejectsMalformedBuilder(t *testing.T) { require.EqualError(t, err, "builder should be in the format: /") } +func TestGetBuilderInstances(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/v2/cloud-builds/accounts/org/builder-groups/builder/instances", r.URL.Path) + assert.Equal(t, "Bearer token", r.Header.Get("Authorization")) + + err := json.NewEncoder(w).Encode(struct { + Results []cloudBuilder `json:"results"` + }{ + Results: []cloudBuilder{{ + Name: "linux-amd64", + Platform: "linux/amd64", + Endpoint: "cloud://org/builder-linux-amd64", + }}, + }) + assert.NoError(t, err) + })) + defer server.Close() + + builders, err := getBuilderInstances(t.Context(), "org/builder", server.URL+"/", "token") + require.NoError(t, err) + require.Len(t, builders, 1) + assert.Equal(t, "linux-amd64", builders[0].Name) + assert.Equal(t, "linux/amd64", builders[0].Platform) + assert.Equal(t, "cloud://org/builder-linux-amd64", builders[0].Endpoint) +} + +func TestGetBuilderInstancesStatusError(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("x-trace-id", "trace-123") + http.Error(w, "denied", http.StatusForbidden) + })) + defer server.Close() + + _, err := getBuilderInstances(t.Context(), "org/builder", server.URL, "token") + require.Error(t, err) + assert.Contains(t, err.Error(), "403 Forbidden") + assert.Contains(t, err.Error(), "trace-123") + assert.Contains(t, err.Error(), "denied") +} + func TestGetBuilderInstanceDriverOpts(t *testing.T) { t.Parallel() From df8f569e1d3c185f4dc618368191a52a4913c5ac Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:00:37 +0200 Subject: [PATCH 08/21] cloud: validate bare internal endpoint hosts Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- driver/cloud/factory.go | 66 ++++++++-- driver/cloud/factory_test.go | 236 +++++++++++++++++++++++++++++++++++ 2 files changed, 292 insertions(+), 10 deletions(-) diff --git a/driver/cloud/factory.go b/driver/cloud/factory.go index aca91f59db25..367aaa28f34f 100644 --- a/driver/cloud/factory.go +++ b/driver/cloud/factory.go @@ -123,21 +123,31 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver if err := verifyDomain(v); err != nil { return nil, err } - d.proxyAddress = v - // registry should not have a protocol, https is implicit - d.registryAddress = strings.TrimPrefix(v, "tcp://") - registryHTTPAddress := strings.Replace(v, "tcp", "https", 1) - d.healthAddress = registryHTTPAddress + "/v2/" + registryAddress, err := endpointHost(v) + if err != nil { + return nil, err + } + d.proxyAddress = "tcp://" + registryAddress + d.registryAddress = registryAddress + d.healthAddress = "https://" + registryAddress + "/v2/" case optKeyInternalProxyAddress: if err := verifyDomain(v); err != nil { return nil, err } - d.proxyAddress = v + proxyAddress, err := endpointHost(v) + if err != nil { + return nil, err + } + d.proxyAddress = "tcp://" + proxyAddress case optKeyInternalRegistryAddress: if err := verifyDomain(v); err != nil { return nil, err } - d.registryAddress = v + registryAddress, err := endpointHost(v) + if err != nil { + return nil, err + } + d.registryAddress = registryAddress case optKeyInternalHealthAddress: if err := verifyDomain(v); err != nil { return nil, err @@ -244,18 +254,54 @@ func getUserContext(env []string) string { } } -func verifyDomain(rawURL string) error { +func parseEndpoint(rawURL string) (*url.URL, error) { u, err := url.Parse(rawURL) + if err == nil && u.Host != "" { + return u, nil + } + + fallback, fallbackErr := url.Parse("//" + rawURL) + if fallbackErr == nil && fallback.Host != "" { + return fallback, nil + } + if err != nil { - return errors.Wrap(err, "could not parse url") + return nil, errors.Wrap(err, "could not parse url") + } + if fallbackErr != nil { + return nil, errors.Wrap(fallbackErr, "could not parse url") } + return u, nil +} + +func endpointHost(rawURL string) (string, error) { + u, err := parseEndpoint(rawURL) + if err != nil { + return "", err + } + host := u.Host + if host == "" { + return "", errors.Errorf("invalid endpoint %s", rawURL) + } + return strings.ToLower(host), nil +} + +func verifyDomain(rawURL string) error { + u, err := parseEndpoint(rawURL) + if err != nil { + return err + } + host := u.Hostname() + if host == "" { + return errors.Errorf("invalid domain %s", rawURL) + } + host = strings.ToLower(host) valid := []string{ "localhost", "docker.io", "docker.com", } - host := u.Hostname() if !strings.ContainsRune(host, '.') { return nil } diff --git a/driver/cloud/factory_test.go b/driver/cloud/factory_test.go index 9d62d58cd9a4..3c6f2d93b9a6 100644 --- a/driver/cloud/factory_test.go +++ b/driver/cloud/factory_test.go @@ -29,6 +29,242 @@ func TestDefaultBuilderName(t *testing.T) { require.EqualError(t, err, "builder should be in the format: /") } +func TestVerifyDomain(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + rawURL string + wantErr bool + }{ + { + name: "DockerURL", + rawURL: "tcp://build-cloud.docker.com:443", + }, + { + name: "MixedCaseDockerURL", + rawURL: "tcp://BUILD-CLOUD.DOCKER.COM:443", + }, + { + name: "DockerHostPort", + rawURL: "build-cloud.docker.com:443", + }, + { + name: "DockerHubURL", + rawURL: "https://index.docker.io/v1/", + }, + { + name: "LocalhostPort", + rawURL: "localhost:5000", + }, + { + name: "SingleLabelHost", + rawURL: "internal-registry", + }, + { + name: "Empty", + rawURL: "", + wantErr: true, + }, + { + name: "ExternalURL", + rawURL: "tcp://example.net:443", + wantErr: true, + }, + { + name: "ExternalHostPort", + rawURL: "example.net:443", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + err := verifyDomain(tt.rawURL) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + }) + } +} + +func TestEndpointHost(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + rawURL string + want string + wantErr bool + }{ + { + name: "TCP", + rawURL: "tcp://build-cloud.docker.com:443", + want: "build-cloud.docker.com:443", + }, + { + name: "HTTPS", + rawURL: "https://build-cloud.docker.com:443", + want: "build-cloud.docker.com:443", + }, + { + name: "BareHostPort", + rawURL: "build-cloud.docker.com:443", + want: "build-cloud.docker.com:443", + }, + { + name: "IPv4", + rawURL: "127.0.0.1:5000", + want: "127.0.0.1:5000", + }, + { + name: "IPv6", + rawURL: "[::1]:5000", + want: "[::1]:5000", + }, + { + name: "MixedCaseDNS", + rawURL: "tcp://BUILD-CLOUD.DOCKER.COM:443", + want: "build-cloud.docker.com:443", + }, + { + name: "Empty", + rawURL: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := endpointHost(tt.rawURL) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestFactoryNewEndpointNormalization(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + driverOpts map[string]string + wantProxy string + wantRegistry string + wantHealth string + }{ + { + name: "InternalAddressTCP", + driverOpts: map[string]string{ + optKeyInternalAddress: "tcp://build-cloud.docker.com:443", + }, + wantProxy: "tcp://build-cloud.docker.com:443", + wantRegistry: "build-cloud.docker.com:443", + wantHealth: "https://build-cloud.docker.com:443/v2/", + }, + { + name: "InternalAddressHTTPS", + driverOpts: map[string]string{ + optKeyInternalAddress: "https://build-cloud.docker.com:443", + }, + wantProxy: "tcp://build-cloud.docker.com:443", + wantRegistry: "build-cloud.docker.com:443", + wantHealth: "https://build-cloud.docker.com:443/v2/", + }, + { + name: "InternalAddressBare", + driverOpts: map[string]string{ + optKeyInternalAddress: "build-cloud.docker.com:443", + }, + wantProxy: "tcp://build-cloud.docker.com:443", + wantRegistry: "build-cloud.docker.com:443", + wantHealth: "https://build-cloud.docker.com:443/v2/", + }, + { + name: "InternalAddressLocalhostBare", + driverOpts: map[string]string{ + optKeyInternalAddress: "localhost:5000", + }, + wantProxy: "tcp://localhost:5000", + wantRegistry: "localhost:5000", + wantHealth: "https://localhost:5000/v2/", + }, + { + name: "InternalProxyAddressHTTPS", + driverOpts: map[string]string{ + optKeyInternalProxyAddress: "https://build-cloud.docker.com:443", + }, + wantProxy: "tcp://build-cloud.docker.com:443", + wantRegistry: RegistryAddress, + wantHealth: HealthAddress, + }, + { + name: "InternalProxyAddressBare", + driverOpts: map[string]string{ + optKeyInternalProxyAddress: "build-cloud.docker.com:443", + }, + wantProxy: "tcp://build-cloud.docker.com:443", + wantRegistry: RegistryAddress, + wantHealth: HealthAddress, + }, + { + name: "InternalProxyAddressIPv6", + driverOpts: map[string]string{ + optKeyInternalProxyAddress: "[::1]:5000", + }, + wantProxy: "tcp://[::1]:5000", + wantRegistry: RegistryAddress, + wantHealth: HealthAddress, + }, + { + name: "InternalProxyAddressMixedCase", + driverOpts: map[string]string{ + optKeyInternalProxyAddress: "tcp://BUILD-CLOUD.DOCKER.COM:443", + }, + wantProxy: "tcp://build-cloud.docker.com:443", + wantRegistry: RegistryAddress, + wantHealth: HealthAddress, + }, + { + name: "InternalRegistryAddress", + driverOpts: map[string]string{ + optKeyInternalRegistryAddress: "https://BUILD-CLOUD.DOCKER.COM:443", + }, + wantProxy: ProxyAddress, + wantRegistry: "build-cloud.docker.com:443", + wantHealth: HealthAddress, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + d, err := (&factory{}).New(t.Context(), driver.InitConfig{ + EndpointAddr: "org/builder", + DriverOpts: tt.driverOpts, + }) + require.NoError(t, err) + + cd, ok := d.(*Driver) + require.True(t, ok) + assert.Equal(t, tt.wantProxy, cd.proxyAddress) + assert.Equal(t, tt.wantRegistry, cd.registryAddress) + assert.Equal(t, tt.wantHealth, cd.healthAddress) + }) + } +} + func TestTokenRefresh(t *testing.T) { t.Parallel() From f89b80a3258463dce901ffb6cec9fd136f71bab4 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:00:43 +0200 Subject: [PATCH 09/21] cloud: require descriptor before cloud pull Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- driver/cloud/driver.go | 24 +++++++++++++----------- driver/cloud/output_test.go | 10 ++++++++++ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/driver/cloud/driver.go b/driver/cloud/driver.go index 95580b9e45df..29ca1231b05e 100644 --- a/driver/cloud/driver.go +++ b/driver/cloud/driver.go @@ -198,6 +198,19 @@ func (d *Driver) HostGatewayIP(ctx context.Context) (net.IP, error) { } func (d *Driver) cloudPull(ctx context.Context, imageDescriptor string, platform string, duc *dockerutil.Client, dockerContext string, tags []string, l progress.SubLogger) error { + if imageDescriptor == "" { + return errors.Errorf("missing exporter response %q", exptypes.ExporterImageDescriptorKey) + } + decodedDescriptor, err := base64.StdEncoding.DecodeString(imageDescriptor) + if err != nil { + return err + } + var desc specs.Descriptor + if err := json.Unmarshal(decodedDescriptor, &desc); err != nil { + return errors.Wrapf(err, "unmarshal descriptor %s", decodedDescriptor) + } + sha := desc.Digest.String() + dc, err := duc.API(dockerContext) if err != nil { return err @@ -233,17 +246,6 @@ func (d *Driver) cloudPull(ctx context.Context, imageDescriptor string, platform Platforms: pullPlatforms, } - // Find manifest to pull - decodedDescriptor, err := base64.StdEncoding.DecodeString(imageDescriptor) - if err != nil { - return err - } - var desc specs.Descriptor - if err := json.Unmarshal(decodedDescriptor, &desc); err != nil { - return errors.Wrapf(err, "unmarshal descriptor %s", decodedDescriptor) - } - sha := desc.Digest.String() - // Use builder name as tag (will be replaced with user tags after the pull) ref := fmt.Sprintf("%s/%s@%s", hydroRegistryAddr, d.builder, sha) diff --git a/driver/cloud/output_test.go b/driver/cloud/output_test.go index aafde43460d2..a3e71f31755f 100644 --- a/driver/cloud/output_test.go +++ b/driver/cloud/output_test.go @@ -391,3 +391,13 @@ func TestCloudPullPlatform(t *testing.T) { }) } } + +func TestCloudPullSessionExporterRequiresDescriptor(t *testing.T) { + t.Parallel() + + provider := (&Driver{}).newCloudPullSessionExporter(driver.PrepareBuildOptions{}, nil, "") + + _, err := provider.FinalizeExport(t.Context(), &sessionexporter.FinalizeExportRequest{}) + require.Error(t, err) + assert.Contains(t, err.Error(), exptypes.ExporterImageDescriptorKey) +} From b8184afb11100b15a2477d1a06b0db5358c16eba Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:13:35 +0200 Subject: [PATCH 10/21] build: test no-output warning behavior Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- build/build.go | 24 +++--- build/build_test.go | 184 +++++++++++++++++++++++++++++++++++++++++++ build/output_test.go | 80 ------------------- 3 files changed, 195 insertions(+), 93 deletions(-) create mode 100644 build/build_test.go delete mode 100644 build/output_test.go diff --git a/build/build.go b/build/build.go index 8aa75335c470..f0badc95982d 100644 --- a/build/build.go +++ b/build/build.go @@ -261,18 +261,6 @@ func findNonMobyDriver(nodes []builder.Node) *driver.DriverHandle { return nil } -func shouldWarnNoOutput(opt Options, reqs []*reqForNode) bool { - if opt.Linked || len(opt.Exports) > 0 { - return false - } - for _, req := range reqs { - if len(req.so.Exports) > 0 || req.so.EnableSessionExporter { - return false - } - } - return true -} - // warnOnNoOutput checks if the prepared build requests would result in an // output and prints a warning if they would not. func warnOnNoOutput(nodes []builder.Node, opts map[string]Options, reqForNodes map[string][]*reqForNode) { @@ -291,7 +279,17 @@ func warnOnNoOutput(nodes []builder.Node, opts map[string]Options, reqForNodes m // Produce a warning describing the targets affected. var noOutputTargets []string for name, opt := range opts { - if shouldWarnNoOutput(opt, reqForNodes[name]) { + if opt.Linked || len(opt.Exports) > 0 { + continue + } + hasOutput := false + for _, req := range reqForNodes[name] { + if len(req.so.Exports) > 0 || req.so.EnableSessionExporter { + hasOutput = true + break + } + } + if !hasOutput { noOutputTargets = append(noOutputTargets, name) } } diff --git a/build/build_test.go b/build/build_test.go new file mode 100644 index 000000000000..d7f65fb4e092 --- /dev/null +++ b/build/build_test.go @@ -0,0 +1,184 @@ +package build + +import ( + "bytes" + "testing" + + "github.com/docker/buildx/builder" + "github.com/docker/buildx/driver" + "github.com/moby/buildkit/client" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type warnOutputFactory struct { + driver.Factory + name string +} + +func (f warnOutputFactory) Name() string { + return f.name +} + +type warnOutputDriver struct { + driver.Driver + factory driver.Factory + moby bool +} + +func (d warnOutputDriver) Factory() driver.Factory { + return d.factory +} + +func (d warnOutputDriver) IsMobyDriver() bool { + return d.moby +} + +func TestWarnOnNoOutput(t *testing.T) { + cloudNodes := []builder.Node{{Driver: newWarnOutputDriver("cloud", false)}} + mobyNodes := []builder.Node{{Driver: newWarnOutputDriver("docker", true)}} + defaultOpts := map[string]Options{"default": {}} + + tests := []struct { + name string + nodes []builder.Node + opts map[string]Options + reqForNodes map[string][]*reqForNode + wantWarning string + }{ + { + name: "NoNonMobyDriver", + opts: defaultOpts, + }, + { + name: "MobyDriver", + nodes: mobyNodes, + opts: defaultOpts, + }, + { + name: "NoRequests", + nodes: cloudNodes, + opts: defaultOpts, + wantWarning: "No output specified with cloud driver.", + }, + { + name: "NoOutput", + nodes: cloudNodes, + opts: defaultOpts, + reqForNodes: reqForDefaultTarget(&client.SolveOpt{}), + wantWarning: "No output specified with cloud driver.", + }, + { + name: "DefaultLoadPreparedExporter", + nodes: cloudNodes, + opts: defaultOpts, + reqForNodes: reqForDefaultTarget(&client.SolveOpt{ + Exports: []client.ExportEntry{{Type: "docker"}}, + }), + }, + { + name: "SessionExporter", + nodes: cloudNodes, + opts: defaultOpts, + reqForNodes: reqForDefaultTarget(&client.SolveOpt{ + EnableSessionExporter: true, + }), + }, + { + name: "OutputOnOneNode", + nodes: cloudNodes, + opts: defaultOpts, + reqForNodes: reqForDefaultTarget( + &client.SolveOpt{}, + &client.SolveOpt{Exports: []client.ExportEntry{{Type: "image"}}}, + ), + }, + { + name: "Linked", + nodes: cloudNodes, + opts: map[string]Options{"default": {Linked: true}}, + }, + { + name: "ExplicitCacheOnly", + nodes: cloudNodes, + opts: map[string]Options{"default": { + Exports: []client.ExportEntry{{Type: "cacheonly"}}, + }}, + }, + { + name: "CallFunc", + nodes: cloudNodes, + opts: map[string]Options{"default": {CallFunc: &CallFunc{Name: "outline"}}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Setenv("BUILDX_NO_DEFAULT_LOAD", "false") + + var buf bytes.Buffer + restoreLogrus := captureLogrusWarnings(&buf) + defer restoreLogrus() + + warnOnNoOutput(tt.nodes, tt.opts, tt.reqForNodes) + + if tt.wantWarning == "" { + assert.Empty(t, buf.String()) + return + } + assert.Contains(t, buf.String(), tt.wantWarning) + assert.Contains(t, buf.String(), "Build result will only remain in the build cache.") + }) + } +} + +func reqForDefaultTarget(solveOpts ...*client.SolveOpt) map[string][]*reqForNode { + reqs := make([]*reqForNode, 0, len(solveOpts)) + for _, so := range solveOpts { + reqs = append(reqs, &reqForNode{so: so}) + } + return map[string][]*reqForNode{"default": reqs} +} + +func newWarnOutputDriver(name string, moby bool) *driver.DriverHandle { + return &driver.DriverHandle{ + Driver: warnOutputDriver{ + factory: warnOutputFactory{name: name}, + moby: moby, + }, + } +} + +func captureLogrusWarnings(w *bytes.Buffer) func() { + logger := logrus.StandardLogger() + oldOut := logger.Out + oldFormatter := logger.Formatter + oldLevel := logger.Level + + logger.SetOutput(w) + logger.SetFormatter(&logrus.TextFormatter{DisableTimestamp: true, DisableColors: true}) + logger.SetLevel(logrus.WarnLevel) + + return func() { + logger.SetOutput(oldOut) + logger.SetFormatter(oldFormatter) + logger.SetLevel(oldLevel) + } +} + +func TestWarnOnNoOutputDisabledByEnv(t *testing.T) { + t.Setenv("BUILDX_NO_DEFAULT_LOAD", "true") + + var buf bytes.Buffer + restoreLogrus := captureLogrusWarnings(&buf) + defer restoreLogrus() + + warnOnNoOutput([]builder.Node{{ + Driver: newWarnOutputDriver("cloud", false), + }}, map[string]Options{ + "default": {}, + }, nil) + + require.Empty(t, buf.String()) +} diff --git a/build/output_test.go b/build/output_test.go deleted file mode 100644 index 8acd30a72726..000000000000 --- a/build/output_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package build - -import ( - "testing" - - "github.com/moby/buildkit/client" - "github.com/stretchr/testify/assert" -) - -func TestShouldWarnNoOutput(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - opt Options - reqs []*reqForNode - wantWarning bool - }{ - { - name: "NoRequests", - wantWarning: true, - }, - { - name: "NoOutput", - reqs: []*reqForNode{{ - so: &client.SolveOpt{}, - }}, - wantWarning: true, - }, - { - name: "Exporter", - reqs: []*reqForNode{{ - so: &client.SolveOpt{ - Exports: []client.ExportEntry{{Type: "image"}}, - }, - }}, - }, - { - name: "SessionExporter", - reqs: []*reqForNode{{ - so: &client.SolveOpt{ - EnableSessionExporter: true, - }, - }}, - }, - { - name: "OutputOnOneNode", - reqs: []*reqForNode{ - { - so: &client.SolveOpt{}, - }, - { - so: &client.SolveOpt{ - Exports: []client.ExportEntry{{Type: "image"}}, - }, - }, - }, - }, - { - name: "Linked", - opt: Options{ - Linked: true, - }, - }, - { - name: "ExplicitCacheOnly", - opt: Options{ - Exports: []client.ExportEntry{{Type: "cacheonly"}}, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - assert.Equal(t, tt.wantWarning, shouldWarnNoOutput(tt.opt, tt.reqs)) - }) - } -} From 4dd66d3097ba1c1d63e9483565d4b3dd407aca1d Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Fri, 14 Aug 2026 17:57:38 +0200 Subject: [PATCH 11/21] cloud: avoid caching token refresh failures Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- driver/cloud/factory_test.go | 83 ++++++++++++++++++++++++++++++++++++ driver/cloud/login.go | 11 +++-- 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/driver/cloud/factory_test.go b/driver/cloud/factory_test.go index 3c6f2d93b9a6..69fdf02768e4 100644 --- a/driver/cloud/factory_test.go +++ b/driver/cloud/factory_test.go @@ -2,6 +2,7 @@ package cloud import ( "context" + "errors" "fmt" "testing" "time" @@ -290,6 +291,88 @@ func TestTokenRefresh(t *testing.T) { assert.Equal(t, "token-2", token) } +func TestTokenRefreshInitialFailureRetries(t *testing.T) { + t.Parallel() + + now := time.Now() + expectedErr := errors.New("refresh failed") + count := 0 + nextToken := newTokenSource(func(context.Context) (string, error) { + count++ + if count == 1 { + return "", expectedErr + } + return "token", nil + }, func() time.Time { return now }) + + token, err := nextToken(t.Context()) + require.ErrorIs(t, err, expectedErr) + assert.Empty(t, token) + + token, err = nextToken(t.Context()) + require.NoError(t, err) + assert.Equal(t, "token", token) + assert.Equal(t, 2, count) +} + +func TestTokenRefreshFailureDoesNotCacheError(t *testing.T) { + t.Parallel() + + now := time.Now() + expectedErr := errors.New("refresh failed") + count := 0 + nextToken := newTokenSource(func(context.Context) (string, error) { + count++ + switch count { + case 1: + return "token-1", nil + case 2: + return "", expectedErr + default: + return "token-3", nil + } + }, func() time.Time { return now }) + + token, err := nextToken(t.Context()) + require.NoError(t, err) + assert.Equal(t, "token-1", token) + + now = now.Add(10 * time.Minute) + token, err = nextToken(t.Context()) + require.ErrorIs(t, err, expectedErr) + assert.Empty(t, token) + + token, err = nextToken(t.Context()) + require.NoError(t, err) + assert.Equal(t, "token-3", token) + assert.Equal(t, 3, count) +} + +func TestTokenRefreshCancellationRetries(t *testing.T) { + t.Parallel() + + now := time.Now() + count := 0 + nextToken := newTokenSource(func(ctx context.Context) (string, error) { + count++ + if err := context.Cause(ctx); err != nil { + return "", err + } + return "token", nil + }, func() time.Time { return now }) + + ctx, cancel := context.WithCancelCause(t.Context()) + cancel(context.Canceled) + token, err := nextToken(ctx) + require.ErrorIs(t, err, context.Canceled) + assert.Empty(t, token) + + token, err = nextToken(t.Context()) + require.NoError(t, err) + assert.Equal(t, "token", token) + assert.Equal(t, 2, count) +} + func TestGetUserContext(t *testing.T) { t.Parallel() diff --git a/driver/cloud/login.go b/driver/cloud/login.go index 140ae447e32f..6562233464de 100644 --- a/driver/cloud/login.go +++ b/driver/cloud/login.go @@ -224,18 +224,21 @@ type tokenSource func(context.Context) (string, error) func newTokenSource(fetch func(context.Context) (string, error), now func() time.Time) tokenSource { var mu sync.Mutex var token string - var err error var lastRefresh time.Time return tokenSource(func(ctx context.Context) (string, error) { mu.Lock() defer mu.Unlock() if lastRefresh.IsZero() || now().Sub(lastRefresh) > tokenRefreshInterval { - lastRefresh = now() ctx, cancel := context.WithTimeoutCause(ctx, 30*time.Second, errors.WithStack(context.DeadlineExceeded)) defer cancel() - token, err = fetch(ctx) + next, err := fetch(ctx) + if err != nil { + return "", err + } + token = next + lastRefresh = now() } - return token, err + return token, nil }) } From 2ba0f5605ce5a557dedfaa039a7008f935d5ad7a Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Mon, 17 Aug 2026 10:15:13 +0200 Subject: [PATCH 12/21] cloud: clean up temporary pull image on errors Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- driver/cloud/driver.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/driver/cloud/driver.go b/driver/cloud/driver.go index 29ca1231b05e..a961add1a850 100644 --- a/driver/cloud/driver.go +++ b/driver/cloud/driver.go @@ -253,6 +253,17 @@ func (d *Driver) cloudPull(ctx context.Context, imageDescriptor string, platform if err != nil { return err } + + removeOpts := dockerclient.ImageRemoveOptions{PruneChildren: false} + defer func() { + cleanupCtx, cancel := context.WithTimeoutCause(context.WithoutCancel(ctx), 30*time.Second, errors.WithStack(context.DeadlineExceeded)) + defer cancel() + + _, err := dc.ImageRemove(cleanupCtx, ref, removeOpts) + if err != nil { + logrus.Debugf("failed to remove temp tag %s: %v", ref, err) + } + }() defer pullResp.Close() if err := printMagicPull(pullResp, l); err != nil { @@ -270,13 +281,6 @@ func (d *Driver) cloudPull(ctx context.Context, imageDescriptor string, platform } } - // TODO: support case where no tags are provided - removeOpts := dockerclient.ImageRemoveOptions{PruneChildren: false} - _, err = dc.ImageRemove(ctx, ref, removeOpts) - if err != nil { - logrus.Debugf("failed to remove temp tag %s: %v", ref, err) - } - return nil } From ceea7f30658d25a19c6c2e407f18d712acd0083c Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Mon, 17 Aug 2026 10:57:02 +0200 Subject: [PATCH 13/21] cloud: simplify registry token pull detection Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- driver/cloud/driver.go | 41 +++++++++---------------- driver/cloud/driver_test.go | 33 ++++++++++---------- util/dockerutil/client.go | 9 ++---- util/dockerutil/features.go | 12 ++++++++ util/dockerutil/features_test.go | 52 ++++++++++++++++++++++++++++++++ 5 files changed, 97 insertions(+), 50 deletions(-) create mode 100644 util/dockerutil/features_test.go diff --git a/driver/cloud/driver.go b/driver/cloud/driver.go index a961add1a850..647386e8686a 100644 --- a/driver/cloud/driver.go +++ b/driver/cloud/driver.go @@ -525,7 +525,6 @@ func daemonSupportsRegistryTokenPull(ctx context.Context, duc *dockerutil.Client if err != nil { return false, err } - resp, err := dc.Info(ctx, dockerclient.InfoOptions{}) if err != nil { return false, err @@ -534,31 +533,19 @@ func daemonSupportsRegistryTokenPull(ctx context.Context, duc *dockerutil.Client } func daemonInfoSupportsRegistryTokenPull(resp system.Info) (bool, error) { - for _, status := range resp.DriverStatus { - if len(status) == 2 && status[1] == "io.containerd.snapshotter.v1" { - // If we're using the containerd snapshotter, we need - // to check for https://github.com/moby/moby/pull/46475. - // - // Simplest way: - // - if we're using docker desktop, assume we have a compatible version - // - otherwise, check if the daemon is on 25.0.0 or newer - constraint, err := semver.NewConstraint(">= 25.0.0") - if err != nil { - return false, err - } - - if resp.OperatingSystem == "Docker Desktop" { - return true, nil - } - - version, err := semver.NewVersion(resp.ServerVersion) - if err != nil { - return false, err - } - - return constraint.Check(version), nil - } + if !dockerutil.HasOCIImporter(resp) { + return true, nil } - - return true, nil + if resp.OperatingSystem == "Docker Desktop" { + return true, nil + } + // If the Docker daemon uses the containerd image store, it needs + // https://github.com/moby/moby/pull/46475 for registry-token pulls. + version, err := semver.NewVersion(resp.ServerVersion) + if err != nil { + // Treat development builds as new enough when the daemon reports the + // required image store. + return true, nil + } + return !version.LessThan(semver.MustParse("25.0.0")), nil } diff --git a/driver/cloud/driver_test.go b/driver/cloud/driver_test.go index 70a52f1e9ac9..0a2bb9d4f69e 100644 --- a/driver/cloud/driver_test.go +++ b/driver/cloud/driver_test.go @@ -2,7 +2,6 @@ package cloud import ( "fmt" - "strings" "testing" "github.com/moby/moby/api/types/system" @@ -15,7 +14,6 @@ type registryTokenSupport struct { version string driver string expected bool - errCheck func(t *testing.T, err error) } func TestRegistryTokenSupport(t *testing.T) { @@ -36,21 +34,28 @@ func TestRegistryTokenSupport(t *testing.T) { }, { os: "Linux Mint 21.1", - version: "24.0.6", + version: "28.0.0-rc.1", + driver: "io.containerd.snapshotter.v1", + expected: true, + }, + { + os: "Linux Mint 21.1", + version: "25.0.0-rc.1", driver: "io.containerd.snapshotter.v1", expected: false, }, { - os: "Linux Mint 21.1", - version: "master", - driver: "io.containerd.snapshotter.v1", - errCheck: func(t *testing.T, err error) { - require.Error(t, err) - // Some versions will return a capitalized error, so we use toLower to normalize the error message - assert.Contains(t, strings.ToLower(err.Error()), "invalid semantic version") - }, + os: "Linux Mint 21.1", + version: "24.0.6", + driver: "io.containerd.snapshotter.v1", expected: false, }, + { + os: "Linux Mint 21.1", + version: "master", + driver: "io.containerd.snapshotter.v1", + expected: true, + }, { os: "Linux Mint 21.1", version: "24.0.6", @@ -70,11 +75,7 @@ func TestRegistryTokenSupport(t *testing.T) { {"driver-type", c.driver}, }, }) - if c.errCheck != nil { - c.errCheck(t, err) - } else { - require.NoError(t, err) - } + require.NoError(t, err) assert.Equal(t, c.expected, actual) }) } diff --git a/util/dockerutil/client.go b/util/dockerutil/client.go index b3b15e2ed429..fa6a81f0e7ab 100644 --- a/util/dockerutil/client.go +++ b/util/dockerutil/client.go @@ -84,13 +84,8 @@ func (c *Client) features(ctx context.Context, name string) map[Feature]bool { features := make(map[Feature]bool) if dapi, err := c.API(name); err == nil { if res, err := dapi.Info(ctx, dockerclient.InfoOptions{}); err == nil { - for _, v := range res.Info.DriverStatus { - switch v[0] { - case "driver-type": - if v[1] == "io.containerd.snapshotter.v1" { - features[OCIImporter] = true - } - } + if HasOCIImporter(res.Info) { + features[OCIImporter] = true } } } diff --git a/util/dockerutil/features.go b/util/dockerutil/features.go index 5d621b0b038a..cd4d837e81ae 100644 --- a/util/dockerutil/features.go +++ b/util/dockerutil/features.go @@ -1,5 +1,17 @@ package dockerutil +import ( + "slices" + + "github.com/moby/moby/api/types/system" +) + type Feature string const OCIImporter Feature = "OCI importer" + +func HasOCIImporter(info system.Info) bool { + return slices.ContainsFunc(info.DriverStatus, func(status [2]string) bool { + return status[0] == "driver-type" && status[1] == "io.containerd.snapshotter.v1" + }) +} diff --git a/util/dockerutil/features_test.go b/util/dockerutil/features_test.go new file mode 100644 index 000000000000..8bc33569eb35 --- /dev/null +++ b/util/dockerutil/features_test.go @@ -0,0 +1,52 @@ +package dockerutil + +import ( + "testing" + + "github.com/moby/moby/api/types/system" + "github.com/stretchr/testify/assert" +) + +func TestHasOCIImporter(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + info system.Info + want bool + }{ + { + name: "ContainerdSnapshotter", + info: system.Info{ + DriverStatus: [][2]string{ + {"driver-type", "io.containerd.snapshotter.v1"}, + }, + }, + want: true, + }, + { + name: "DockerDriver", + info: system.Info{ + DriverStatus: [][2]string{ + {"driver-type", "docker"}, + }, + }, + }, + { + name: "UnrelatedStatus", + info: system.Info{ + DriverStatus: [][2]string{ + {"Backing Filesystem", "extfs"}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.want, HasOCIImporter(tt.info)) + }) + } +} From 5a7fab5c51d9a0ffb2d52ac636b62b79b1c08b8f Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Mon, 17 Aug 2026 11:01:31 +0200 Subject: [PATCH 14/21] cloud: clean up driver error and header handling Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- driver/cloud/driver.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/driver/cloud/driver.go b/driver/cloud/driver.go index 647386e8686a..167be98d923c 100644 --- a/driver/cloud/driver.go +++ b/driver/cloud/driver.go @@ -92,7 +92,7 @@ func (d *Driver) Info(ctx context.Context) (*driver.Info, error) { tlsConfig, err := d.tlsConfig() if err != nil { - return nil, errors.Wrapf(err, "failed to create TLS config: %v", err) + return nil, errors.Wrap(err, "failed to create TLS config") } client := newClient(&http.Transport{ TLSClientConfig: tlsConfig, @@ -142,7 +142,7 @@ func (d *Driver) Client(ctx context.Context, opts ...client.ClientOpt) (*client. if err != nil { return errors.Wrap(err, "fetching auth token") } - headers := append(d.headers, authorizationKey, "Bearer "+token) + headers := slices.Concat(d.headers, []string{authorizationKey, "Bearer " + token}) ctx = metadata.AppendToOutgoingContext(ctx, headers...) opts = append([]grpc.CallOption{grpc.UseCompressor(grpcgzip.Name)}, opts...) return invoker(ctx, method, req, reply, cc, opts...) @@ -152,7 +152,7 @@ func (d *Driver) Client(ctx context.Context, opts ...client.ClientOpt) (*client. if err != nil { return nil, errors.Wrap(err, "fetching auth token") } - headers := append(d.headers, authorizationKey, "Bearer "+token) + headers := slices.Concat(d.headers, []string{authorizationKey, "Bearer " + token}) ctx = metadata.AppendToOutgoingContext(ctx, headers...) opts = append([]grpc.CallOption{grpc.UseCompressor(grpcgzip.Name)}, opts...) return streamer(ctx, desc, cc, method, opts...) @@ -428,7 +428,7 @@ func (d *Driver) tlsConfig() (*tls.Config, error) { } ca, err := os.ReadFile(d.tls.caCert) if err != nil { - return nil, errors.Wrapf(err, "could not read CA certificate %s: %v", d.tls.caCert, err) + return nil, errors.Wrapf(err, "could not read CA certificate %s", d.tls.caCert) } if ok := rootCAs.AppendCertsFromPEM(ca); !ok { return nil, errors.Errorf("failed to add CA certificate %s to root CAs", d.tls.caCert) From 1da20d63f8a7316cc685719138cb135fa79efeec Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Mon, 17 Aug 2026 11:16:56 +0200 Subject: [PATCH 15/21] cloud: surface health check failures Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- driver/cloud/driver.go | 24 ++++++++---- driver/cloud/driver_test.go | 78 +++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 8 deletions(-) diff --git a/driver/cloud/driver.go b/driver/cloud/driver.go index 167be98d923c..d30c8f78a66a 100644 --- a/driver/cloud/driver.go +++ b/driver/cloud/driver.go @@ -94,18 +94,25 @@ func (d *Driver) Info(ctx context.Context) (*driver.Info, error) { if err != nil { return nil, errors.Wrap(err, "failed to create TLS config") } - client := newClient(&http.Transport{ + c := newClient(&http.Transport{ TLSClientConfig: tlsConfig, }) - resp, err := client.Do(req) - if resp != nil { - defer drainResponse(resp) + resp, err := c.Do(req) + if err != nil { + if resp != nil { + drainResponse(resp) + } + return nil, errors.Wrap(err, "cloud registry health check failed") + } + if resp == nil { + return nil, errors.New("cloud registry health check failed: empty response") } - if err != nil || resp.StatusCode != http.StatusUnauthorized { - return &driver.Info{ - Status: driver.Inactive, - }, nil + defer drainResponse(resp) + if resp.StatusCode != http.StatusUnauthorized { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 255)) + return nil, errors.Errorf("cloud registry health check failed, got: %s. Trace ID: %s. Response body: %s", + resp.Status, resp.Header.Get("x-trace-id"), string(body)) } return &driver.Info{ @@ -418,6 +425,7 @@ func resolveCloudPullExporters(exports []client.ExportEntry, tags []string, buil func (d *Driver) tlsConfig() (*tls.Config, error) { tlsConfig := &tls.Config{ InsecureSkipVerify: d.tls.insecure, + ServerName: d.tls.serverName, } if d.tls.caCert != "" { // Prefer the system cert pool if available, but fallback diff --git a/driver/cloud/driver_test.go b/driver/cloud/driver_test.go index 0a2bb9d4f69e..d89193b382c6 100644 --- a/driver/cloud/driver_test.go +++ b/driver/cloud/driver_test.go @@ -1,14 +1,92 @@ package cloud import ( + "encoding/pem" "fmt" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" "testing" + "github.com/docker/buildx/driver" "github.com/moby/moby/api/types/system" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func TestInfo(t *testing.T) { + t.Parallel() + + t.Run("Running", func(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + })) + t.Cleanup(server.Close) + + info, err := (&Driver{healthAddress: server.URL}).Info(t.Context()) + + require.NoError(t, err) + require.NotNil(t, info) + assert.Equal(t, driver.Running, info.Status) + }) + + t.Run("StatusError", func(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("x-trace-id", "trace-123") + http.Error(w, "denied", http.StatusForbidden) + })) + t.Cleanup(server.Close) + + info, err := (&Driver{healthAddress: server.URL}).Info(t.Context()) + + require.Error(t, err) + assert.Nil(t, info) + assert.Contains(t, err.Error(), "403 Forbidden") + assert.Contains(t, err.Error(), "trace-123") + assert.Contains(t, err.Error(), "denied") + }) + + t.Run("TLSServerName", func(t *testing.T) { + t.Parallel() + + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + })) + t.Cleanup(server.Close) + + caPath := filepath.Join(t.TempDir(), "ca.pem") + certPEM := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: server.Certificate().Raw, + }) + require.NoError(t, os.WriteFile(caPath, certPEM, 0600)) + + serverURL, err := url.Parse(server.URL) + require.NoError(t, err) + _, port, err := net.SplitHostPort(serverURL.Host) + require.NoError(t, err) + + info, err := (&Driver{ + healthAddress: "https://localhost:" + port, + tls: tlsOpts{ + caCert: caPath, + serverName: "example.com", + }, + }).Info(t.Context()) + + require.NoError(t, err) + require.NotNil(t, info) + assert.Equal(t, driver.Running, info.Status) + }) +} + type registryTokenSupport struct { os string version string From a58e706a82986faa19af66744ffc91baed57f1fb Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Mon, 17 Aug 2026 11:22:03 +0200 Subject: [PATCH 16/21] cloud: honor retry-after on retries Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- driver/cloud/login.go | 14 ++++++++++++++ driver/cloud/login_test.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/driver/cloud/login.go b/driver/cloud/login.go index 6562233464de..bed14b221a7c 100644 --- a/driver/cloud/login.go +++ b/driver/cloud/login.go @@ -9,6 +9,7 @@ import ( "net/http" "net/url" "os" + "strconv" "sync" "time" @@ -188,6 +189,19 @@ func (c *retryClient) Do(req *http.Request) (*http.Response, error) { } wait := min(retryWaitInitial<= 0 { + wait = min(time.Duration(seconds)*time.Second, retryWaitMax) + } + } else if t, err := http.ParseTime(retryAfter); err == nil { + if d := time.Until(t); d > 0 { + wait = min(d, retryWaitMax) + } + } + } + } timer := time.NewTimer(wait) select { case <-req.Context().Done(): diff --git a/driver/cloud/login_test.go b/driver/cloud/login_test.go index c3f7263f7a59..3dec9e97248c 100644 --- a/driver/cloud/login_test.go +++ b/driver/cloud/login_test.go @@ -55,3 +55,37 @@ func TestRetryClient(t *testing.T) { assert.Equal(t, wantWait, time.Since(start)) }) } + +func TestRetryClientRetryAfter(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + var attempts int + transport := roundTripperFunc(func(req *http.Request) (*http.Response, error) { + attempts++ + if attempts == 1 { + return &http.Response{ + StatusCode: http.StatusTooManyRequests, + Header: http.Header{"Retry-After": []string{"2"}}, + Body: http.NoBody, + }, nil + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: make(http.Header), + Body: http.NoBody, + }, nil + }) + + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "http://example.com", nil) + require.NoError(t, err) + + start := time.Now() + resp, err := newClient(transport).Do(req) + require.NoError(t, err) + defer resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, 2, attempts) + assert.Equal(t, 2*time.Second, time.Since(start)) + }) +} From 9fa592795f67f5d5a99fba899acf9691b8ee1a3c Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Mon, 17 Aug 2026 11:33:00 +0200 Subject: [PATCH 17/21] desktop: expose buildx version fallback Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- driver/cloud/factory.go | 26 ++------------------- util/desktop/version.go | 32 ++++++++++++++++++++++++++ util/desktop/version_test.go | 44 ++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 24 deletions(-) create mode 100644 util/desktop/version.go create mode 100644 util/desktop/version_test.go diff --git a/driver/cloud/factory.go b/driver/cloud/factory.go index 367aaa28f34f..ad26a82dd45f 100644 --- a/driver/cloud/factory.go +++ b/driver/cloud/factory.go @@ -5,7 +5,6 @@ import ( "net/url" "os" "runtime" - "runtime/debug" "slices" "strconv" "strings" @@ -13,7 +12,7 @@ import ( "time" "github.com/docker/buildx/driver" - "github.com/docker/buildx/version" + "github.com/docker/buildx/util/desktop" dockerclient "github.com/moby/moby/client" "github.com/pkg/errors" ) @@ -200,7 +199,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver func (f *factory) prepareHeaders() { f.once.Do(func() { f.headers = []string{ - buildxVersionKey, getBuildxVersion(), + buildxVersionKey, desktop.BuildxVersion(), userContextKey, getUserContext(os.Environ()), userPlatformKey, runtime.GOOS + "/" + runtime.GOARCH, } @@ -211,27 +210,6 @@ func (f *factory) AllowsInstances() bool { return true } -// When called from desktop build backend, the version.Version is unknown. We try to get the version from the build info. -func getBuildxVersion() string { - if version.Version != "v0.0.0+unknown" { - return version.Version - } - info, ok := debug.ReadBuildInfo() - if !ok { - return version.Version - } - for _, dep := range info.Deps { - if dep.Path == "github.com/docker/buildx" { - if dep.Replace != nil { - return dep.Replace.Version - } - return dep.Version - } - } - - return version.Version -} - func getUserContext(env []string) string { if !slices.Contains(env, "CI=true") { return "local" diff --git a/util/desktop/version.go b/util/desktop/version.go new file mode 100644 index 000000000000..9a39834a9dd8 --- /dev/null +++ b/util/desktop/version.go @@ -0,0 +1,32 @@ +package desktop + +import ( + "runtime/debug" + "sync" + + "github.com/docker/buildx/version" +) + +var readBuildInfo = sync.OnceValues(debug.ReadBuildInfo) + +func BuildxVersion() string { + if version.Version != "v0.0.0+unknown" { + return version.Version + } + info, ok := readBuildInfo() + if !ok || info == nil { + return version.Version + } + for _, dep := range info.Deps { + if dep.Path != version.Package { + continue + } + if dep.Replace != nil && dep.Replace.Version != "" { + return dep.Replace.Version + } + if dep.Version != "" { + return dep.Version + } + } + return version.Version +} diff --git a/util/desktop/version_test.go b/util/desktop/version_test.go new file mode 100644 index 000000000000..157af6103828 --- /dev/null +++ b/util/desktop/version_test.go @@ -0,0 +1,44 @@ +package desktop + +import ( + "runtime/debug" + "sync" + "testing" + + "github.com/docker/buildx/version" + "github.com/stretchr/testify/assert" +) + +func TestBuildxVersionLinked(t *testing.T) { + oldVersion := version.Version + t.Cleanup(func() { + version.Version = oldVersion + }) + version.Version = "v1.2.3" + assert.Equal(t, "v1.2.3", BuildxVersion()) +} + +func TestBuildxVersionBuildInfo(t *testing.T) { + oldVersion := version.Version + oldReadBuildInfo := readBuildInfo + t.Cleanup(func() { + version.Version = oldVersion + readBuildInfo = oldReadBuildInfo + }) + + var calls int + version.Version = "v0.0.0+unknown" + readBuildInfo = sync.OnceValues(func() (*debug.BuildInfo, bool) { + calls++ + return &debug.BuildInfo{ + Deps: []*debug.Module{{ + Path: version.Package, + Version: "v1.2.3", + }}, + }, true + }) + + assert.Equal(t, "v1.2.3", BuildxVersion()) + assert.Equal(t, "v1.2.3", BuildxVersion()) + assert.Equal(t, 1, calls) +} From 596f1b109aae24974b16f8b896459c1c9b1997a0 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Mon, 17 Aug 2026 16:56:06 +0200 Subject: [PATCH 18/21] cloud: wrap external client errors Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- driver/cloud/driver.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/driver/cloud/driver.go b/driver/cloud/driver.go index d30c8f78a66a..761f16768e38 100644 --- a/driver/cloud/driver.go +++ b/driver/cloud/driver.go @@ -172,7 +172,7 @@ func (d *Driver) Client(ctx context.Context, opts ...client.ClientOpt) (*client. ) c, err := client.New(ctx, d.proxyAddress, opts...) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "connecting to Docker Build Cloud at %s", d.proxyAddress) } return c, nil } @@ -220,7 +220,7 @@ func (d *Driver) cloudPull(ctx context.Context, imageDescriptor string, platform dc, err := duc.API(dockerContext) if err != nil { - return err + return errors.Wrapf(err, "resolving Docker context %q", dockerContext) } token, err := d.tokenSource(ctx) @@ -258,7 +258,7 @@ func (d *Driver) cloudPull(ctx context.Context, imageDescriptor string, platform pullResp, err := dc.ImagePull(ctx, ref, pullOpts) if err != nil { - return err + return errors.Wrapf(err, "pulling temporary cloud image %s", ref) } removeOpts := dockerclient.ImageRemoveOptions{PruneChildren: false} @@ -274,7 +274,7 @@ func (d *Driver) cloudPull(ctx context.Context, imageDescriptor string, platform defer pullResp.Close() if err := printMagicPull(pullResp, l); err != nil { - return err + return errors.Wrap(err, "reading cloud pull progress") } // Tag image with user tags and remove the builder name tag @@ -284,7 +284,7 @@ func (d *Driver) cloudPull(ctx context.Context, imageDescriptor string, platform Target: tag, } if _, err = dc.ImageTag(ctx, opts); err != nil { - return err + return errors.Wrapf(err, "tagging cloud image %s as %s", ref, tag) } } @@ -531,11 +531,11 @@ func printMagicPull(rc io.Reader, l progress.SubLogger) (err error) { func daemonSupportsRegistryTokenPull(ctx context.Context, duc *dockerutil.Client, dockerContext string) (bool, error) { dc, err := duc.API(dockerContext) if err != nil { - return false, err + return false, errors.Wrapf(err, "resolving Docker context %q", dockerContext) } resp, err := dc.Info(ctx, dockerclient.InfoOptions{}) if err != nil { - return false, err + return false, errors.Wrap(err, "querying Docker daemon info") } return daemonInfoSupportsRegistryTokenPull(resp.Info) } From bade1fa7c0bd20dd79b2e2124b2fe6ad993a31b7 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:04:00 +0200 Subject: [PATCH 19/21] docs: document cloud driver create behavior Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- docs/reference/buildx_create.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/reference/buildx_create.md b/docs/reference/buildx_create.md index 8032223fbe4b..47e574fc7c7e 100644 --- a/docs/reference/buildx_create.md +++ b/docs/reference/buildx_create.md @@ -117,6 +117,7 @@ Sets the builder driver to be used. A driver is a configuration of a BuildKit backend. Buildx supports the following drivers: * `docker` (default) +* `cloud` * `docker-container` * `kubernetes` * `remote` @@ -130,6 +131,34 @@ the [`--load`](buildx_build.md#load) flag is implied by default on `buildx build`. However, building multi-platform images or exporting cache is not currently supported. +#### `cloud` driver + +Uses Docker Build Cloud builders. For Docker Build Cloud access and setup, see +[Docker Build Cloud setup](https://docs.docker.com/build-cloud/setup/). For +build examples, see [Building with Docker Build Cloud](https://docs.docker.com/build-cloud/usage/). + +Connect Buildx to an existing cloud builder with `--driver cloud` and the +builder name in the form `/` or +`cloud:///`. You must be signed in with `docker login` and +have access to the builder. + +```console +$ docker buildx create --driver cloud --name cloud-builder --use org/builder +$ docker buildx create --driver cloud --name cloud-builder --use cloud://org/builder +``` + +When you don't specify an output, an untagged result remains in the cloud build +cache. If you use `--tag`, Buildx automatically loads the image when the build +targets a single platform and runs on one cloud node. For details about loading +behavior, Docker contexts, and other output configurations, see +[Load results from Docker Build Cloud](https://docs.docker.com/build/exporters/). + +When `buildx create` resolves a Build Cloud builder group, each service instance +becomes a Buildx node. `--append` requires the resolved instance names to be +unique in the Buildx builder. Resolved cloud nodes are a create-time snapshot; +recreate the Buildx builder to refresh topology after Build Cloud membership or +placement changes. + #### `docker-container` driver Uses a BuildKit container that will be spawned via Docker. With this driver, From f9be50c289902b2e4508fd29dd9103e9c0ad9e83 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Tue, 18 Aug 2026 10:18:05 +0200 Subject: [PATCH 20/21] docs: add cloud driver project metadata Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/labeler.yml | 5 +++++ PROJECT.md | 1 + README.md | 9 ++++++--- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index 49336dc51316..aca011b205ad 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -73,6 +73,11 @@ area/driver: - changed-files: - any-glob-to-any-file: 'driver/**' +# Add 'area/driver/cloud' label to changes in the cloud driver +area/driver/cloud: + - changed-files: + - any-glob-to-any-file: 'driver/cloud/**' + # Add 'area/driver/docker' label to changes in the docker driver area/driver/docker: - changed-files: diff --git a/PROJECT.md b/PROJECT.md index 2c318d67c7f4..f5d081e26f35 100644 --- a/PROJECT.md +++ b/PROJECT.md @@ -84,6 +84,7 @@ Area or component of the project affected. Please note that the table below may | `area/dockerfile` | Any | `dockerfile` | | `area/docs` | Any | `docs` | | `area/driver` | Any | `driver` | +| `area/driver/cloud` | Any | `driver/cloud` | | `area/driver/docker` | Any | `driver/docker` | | `area/driver/docker-container` | Any | `driver/docker-container` | | `area/driver/kubernetes` | Any | `driver/kubernetes` | diff --git a/README.md b/README.md index 3e42584b90d7..65b9ccef080e 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,7 @@ feature sets. We currently support the following drivers: - The `docker` driver ([manual](https://docs.docker.com/build/builders/drivers/docker/)) +- The `cloud` driver ([manual](https://docs.docker.com/build-cloud/)) - The `docker-container` driver ([manual](https://docs.docker.com/build/builders/drivers/docker-container/)) - The `kubernetes` driver ([manual](https://docs.docker.com/build/drivers/kubernetes/)) - The `remote` driver ([manual](https://docs.docker.com/build/builders/drivers/remote/)) @@ -217,9 +218,11 @@ When you invoke a build, you can set the `--platform` flag to specify the target platform for the build output, (for example, `linux/amd64`, `linux/arm64`, or `darwin/amd64`). -When the current builder instance is backed by the `docker-container` or -`kubernetes` driver, you can specify multiple platforms together. In this case, -it builds a manifest list which contains images for all specified architectures. +When the current builder instance is backed by the `cloud`, `docker-container`, +`kubernetes` or `remote` driver, you can specify multiple platforms together. +In this case, it builds a manifest list which contains images for all specified +architectures. + When you use this image in [`docker run`](https://docs.docker.com/reference/cli/docker/container/run/) or [`docker service`](https://docs.docker.com/reference/cli/docker/service/), Docker picks the correct image based on the node's platform. From 53e97e4631c7187ff123b9a701ee468c93447295 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:47:44 +0200 Subject: [PATCH 21/21] dockerutil: share docker progress decoding Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- driver/cloud/driver.go | 85 +----------------------- util/dockerutil/client.go | 2 +- util/dockerutil/progress.go | 69 ++++++++++++++++--- util/dockerutil/progress_test.go | 109 +++++++++++++++++++++++++++++++ 4 files changed, 172 insertions(+), 93 deletions(-) create mode 100644 util/dockerutil/progress_test.go diff --git a/driver/cloud/driver.go b/driver/cloud/driver.go index 761f16768e38..c2e61f75eea4 100644 --- a/driver/cloud/driver.go +++ b/driver/cloud/driver.go @@ -27,7 +27,6 @@ import ( "github.com/moby/buildkit/exporter/containerimage/exptypes" sessionexporter "github.com/moby/buildkit/session/exporter" "github.com/moby/buildkit/session/exporter/exporterprovider" - "github.com/moby/moby/api/types/jsonstream" "github.com/moby/moby/api/types/registry" "github.com/moby/moby/api/types/system" dockerclient "github.com/moby/moby/client" @@ -273,7 +272,7 @@ func (d *Driver) cloudPull(ctx context.Context, imageDescriptor string, platform }() defer pullResp.Close() - if err := printMagicPull(pullResp, l); err != nil { + if err := dockerutil.PullProgressFromReader(l, pullResp); err != nil { return errors.Wrap(err, "reading cloud pull progress") } @@ -446,88 +445,6 @@ func (d *Driver) tlsConfig() (*tls.Config, error) { return tlsConfig, nil } -func printMagicPull(rc io.Reader, l progress.SubLogger) (err error) { - started := map[string]client.VertexStatus{} - - defer func() { - if err != nil { - return - } - for _, st := range started { - if st.Completed == nil { - now := time.Now() - st.Completed = &now - l.SetStatus(&st) - } - } - }() - - dec := json.NewDecoder(rc) - - var ( - parsedError error - jm jsonstream.Message - ) - - for { - if err = dec.Decode(&jm); err != nil { - if parsedError != nil { - err = parsedError - } else if err == io.EOF { - err = nil - } - return - } - - if jm.Error != nil { - parsedError = jm.Error - } - - if jm.ID == "" { - continue - } - - // handle temporary fake registry tags - if strings.ContainsRune(jm.ID, '@') { - continue - } - if strings.ContainsRune(jm.Status, ':') { - continue - } - - id := "pulling layer " + jm.ID - st, ok := started[id] - if !ok { - if jm.Progress != nil || strings.HasPrefix(jm.Status, "Pulling") || strings.HasPrefix(jm.Status, "Already exists") { - now := time.Now() - st = client.VertexStatus{ - ID: id, - Started: &now, - } - } else { - continue - } - } - st.Timestamp = time.Now() - if jm.Progress != nil && jm.Status == "Downloading" { - st.Current = jm.Progress.Current - st.Total = jm.Progress.Total - } - if jm.Error != nil { - now := time.Now() - st.Completed = &now - } - - if jm.Status == "Pull complete" || jm.Status == "Already exists" { - now := time.Now() - st.Completed = &now - st.Current = st.Total - } - started[id] = st - l.SetStatus(&st) - } -} - func daemonSupportsRegistryTokenPull(ctx context.Context, duc *dockerutil.Client, dockerContext string) (bool, error) { dc, err := duc.API(dockerContext) if err != nil { diff --git a/util/dockerutil/client.go b/util/dockerutil/client.go index fa6a81f0e7ab..68ff9a29c281 100644 --- a/util/dockerutil/client.go +++ b/util/dockerutil/client.go @@ -61,7 +61,7 @@ func (c *Client) LoadImage(ctx context.Context, name string, status progress.Wri status = progress.ResetTime(status) if err := progress.Wrap("importing to docker", status.Write, func(l progress.SubLogger) error { - return fromReader(l, resp) + return loadProgressFromReader(l, resp) }); err != nil { handleErr(err) } diff --git a/util/dockerutil/progress.go b/util/dockerutil/progress.go index a8c0d03d812f..a68082f3433f 100644 --- a/util/dockerutil/progress.go +++ b/util/dockerutil/progress.go @@ -3,6 +3,7 @@ package dockerutil import ( "encoding/json" "io" + "strings" "time" "github.com/docker/buildx/util/progress" @@ -12,10 +13,32 @@ import ( const minTimeDelta = 2 * time.Second -func fromReader(l progress.SubLogger, rc io.ReadCloser) error { +type progressKind int + +const ( + progressKindLoad progressKind = iota + progressKindPull +) + +// loadProgressFromReader parses Docker image load JSON progress and forwards it +// to a BuildKit progress sublogger. +func loadProgressFromReader(l progress.SubLogger, rc io.ReadCloser) error { + return progressFromReader(l, rc, progressKindLoad) +} + +// PullProgressFromReader parses Docker image pull JSON progress and forwards it +// to a BuildKit progress sublogger. +func PullProgressFromReader(l progress.SubLogger, rc io.Reader) error { + return progressFromReader(l, rc, progressKindPull) +} + +func progressFromReader(l progress.SubLogger, rc io.Reader, kind progressKind) (retErr error) { started := map[string]client.VertexStatus{} defer func() { + if retErr != nil && kind != progressKindLoad { + return + } for _, st := range started { if st.Completed == nil { now := time.Now() @@ -27,41 +50,71 @@ func fromReader(l progress.SubLogger, rc io.ReadCloser) error { dec := json.NewDecoder(rc) var parsedErr error - var jm jsonstream.Message for { + var jm jsonstream.Message if err := dec.Decode(&jm); err != nil { if parsedErr != nil { - return parsedErr + retErr = parsedErr + return retErr } if err == io.EOF { break } - return err + retErr = err + return retErr } if jm.Error != nil { parsedErr = jm.Error } - if jm.ID == "" || jm.Progress == nil { + if jm.ID == "" { + continue + } + + var id string + var start, complete, updateProgress bool + switch kind { + case progressKindLoad: + if jm.Progress == nil { + continue + } + id = "loading layer " + jm.ID + start = true + updateProgress = jm.Status == "Loading layer" + case progressKindPull: + if strings.ContainsRune(jm.ID, '@') || strings.ContainsRune(jm.Status, ':') || strings.HasPrefix(jm.Status, "Pulling from ") { + continue + } + id = "pulling layer " + jm.ID + start = jm.Progress != nil || strings.HasPrefix(jm.Status, "Pulling") || strings.HasPrefix(jm.Status, "Already exists") + complete = jm.Status == "Pull complete" || jm.Status == "Already exists" + updateProgress = jm.Progress != nil && jm.Status == "Downloading" + } + if !start && !complete { continue } - id := "loading layer " + jm.ID st, ok := started[id] if !ok { + if !start { + continue + } now := time.Now() st = client.VertexStatus{ ID: id, Started: &now, } } - if jm.Status == "Loading layer" { + if updateProgress { st.Current = jm.Progress.Current st.Total = jm.Progress.Total } now := time.Now() if jm.Error != nil { st.Completed = &now - } else { + } else if complete { + st.Completed = &now + st.Current = st.Total + } else if kind == progressKindLoad { timeDelta := time.Since(st.Timestamp) if timeDelta < minTimeDelta { started[id] = st diff --git a/util/dockerutil/progress_test.go b/util/dockerutil/progress_test.go new file mode 100644 index 000000000000..0d8490e2f297 --- /dev/null +++ b/util/dockerutil/progress_test.go @@ -0,0 +1,109 @@ +package dockerutil + +import ( + "io" + "strings" + "testing" + + "github.com/moby/buildkit/client" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadProgressIgnoresStaleFields(t *testing.T) { + t.Parallel() + + logger := &captureSubLogger{} + err := loadProgressFromReader(logger, io.NopCloser(strings.NewReader(` +{"id":"layer1","status":"Loading layer","progressDetail":{"current":5,"total":10}} +{"id":"layer2","status":"Loading layer"} +`))) + require.NoError(t, err) + + require.Len(t, logger.statuses, 2) + assert.Equal(t, "loading layer layer1", logger.statuses[0].ID) + assert.Equal(t, int64(5), logger.statuses[0].Current) + assert.Equal(t, int64(10), logger.statuses[0].Total) + assert.Nil(t, logger.statuses[0].Completed) + assert.Equal(t, "loading layer layer1", logger.statuses[1].ID) + assert.NotNil(t, logger.statuses[1].Completed) + assert.Equal(t, int64(5), logger.statuses[1].Current) + assert.NotContains(t, statusIDs(logger.statuses), "loading layer layer2") +} + +func TestPullProgressIgnoresStaleFields(t *testing.T) { + t.Parallel() + + logger := &captureSubLogger{} + err := PullProgressFromReader(logger, strings.NewReader(` +{"id":"layer1","status":"Downloading","progressDetail":{"current":5,"total":10}} +{"status":"Pull complete"} +{"id":"layer2","status":"Downloading"} +`)) + require.NoError(t, err) + + require.Len(t, logger.statuses, 2) + assert.Equal(t, "pulling layer layer1", logger.statuses[0].ID) + assert.Equal(t, int64(5), logger.statuses[0].Current) + assert.Equal(t, int64(10), logger.statuses[0].Total) + assert.Nil(t, logger.statuses[0].Completed) + assert.Equal(t, "pulling layer layer1", logger.statuses[1].ID) + assert.NotNil(t, logger.statuses[1].Completed) + assert.Equal(t, int64(5), logger.statuses[1].Current) + assert.NotContains(t, statusIDs(logger.statuses), "pulling layer layer2") +} + +func TestPullProgressKeepsCountersOnCompletion(t *testing.T) { + t.Parallel() + + logger := &captureSubLogger{} + err := PullProgressFromReader(logger, strings.NewReader(` +{"id":"layer1","status":"Downloading","progressDetail":{"current":5,"total":10}} +{"id":"layer1","status":"Download complete","progressDetail":{}} +{"id":"layer1","status":"Pull complete","progressDetail":{}} +`)) + require.NoError(t, err) + + require.Len(t, logger.statuses, 3) + assert.Equal(t, "pulling layer layer1", logger.statuses[2].ID) + assert.Equal(t, int64(10), logger.statuses[2].Current) + assert.Equal(t, int64(10), logger.statuses[2].Total) + assert.NotNil(t, logger.statuses[2].Completed) +} + +func TestPullProgressIgnoresPullingFrom(t *testing.T) { + t.Parallel() + + logger := &captureSubLogger{} + err := PullProgressFromReader(logger, strings.NewReader(` +{"id":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","status":"Pulling from library/alpine"} +{"id":"layer1","status":"Pulling fs layer"} +`)) + require.NoError(t, err) + + assert.NotContains(t, statusIDs(logger.statuses), "pulling layer sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + assert.Contains(t, statusIDs(logger.statuses), "pulling layer layer1") +} + +type captureSubLogger struct { + statuses []*client.VertexStatus +} + +func (l *captureSubLogger) Wrap(name string, fn func() error) error { + return fn() +} + +func (l *captureSubLogger) Log(stream int, dt []byte) {} + +func (l *captureSubLogger) SetStatus(st *client.VertexStatus) { + cp := *st + l.statuses = append(l.statuses, &cp) +} + +func statusIDs(statuses []*client.VertexStatus) []string { + out := make([]string, 0, len(statuses)) + for _, st := range statuses { + out = append(out, st.ID) + } + return out +}