From c4b060186c6c81ae322619595f40d430f583b85f Mon Sep 17 00:00:00 2001 From: Burak Varli Date: Fri, 20 Nov 2026 19:24:40 +0000 Subject: [PATCH] objectstorage: add request id and error code to S3's ServiceError `RequestId` and `Code` from S3 errors were ignored previously and logs were unhelpful to understand what failed when uploading snapshots to S3. With this change, we now get those in the logs. Signed-off-by: Burak Varli --- internal/objectstoreplugin/status.go | 3 ++ pkg/objectstorage/objects.go | 9 +++- pkg/objectstorage/s3.go | 45 +++++++++++++++++ pkg/objectstorage/s3_test.go | 73 ++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 2 deletions(-) diff --git a/internal/objectstoreplugin/status.go b/internal/objectstoreplugin/status.go index 607931f971..86c33f6498 100644 --- a/internal/objectstoreplugin/status.go +++ b/internal/objectstoreplugin/status.go @@ -64,6 +64,9 @@ func internalErrorAttrs(err error) []any { if desc.Message != "" { attrs = append(attrs, slog.String("message", desc.Message)) } + if desc.RequestID != "" { + attrs = append(attrs, slog.String("request_id", desc.RequestID)) + } } for next := errors.Unwrap(err); next != nil; next = errors.Unwrap(err) { err = next diff --git a/pkg/objectstorage/objects.go b/pkg/objectstorage/objects.go index 4df1e4f1fb..97e4f50c70 100644 --- a/pkg/objectstorage/objects.go +++ b/pkg/objectstorage/objects.go @@ -48,17 +48,22 @@ type ServiceError struct { // Message is the service's own description, if the backend reports one // it knows names no caller, resource or credential. Message string + // RequestID is the ID the service gave the request, if the backend + // reports one, so the service's operator can find it. + RequestID string } // DescribeServiceError describes the storage service response that err // carries, if any. Each backend decides what of its errors is safe to // describe; an error in err's chain from any other source with an -// HTTPStatusCode() int method, as the S3 SDK's response errors have, is -// described by its status alone. +// HTTPStatusCode() int method is described by its status alone. func DescribeServiceError(err error) (ServiceError, bool) { if desc, ok := describeGCSError(err); ok { return desc, true } + if desc, ok := describeS3Error(err); ok { + return desc, true + } var withStatus interface{ HTTPStatusCode() int } if errors.As(err, &withStatus) { return ServiceError{HTTPStatus: withStatus.HTTPStatusCode()}, true diff --git a/pkg/objectstorage/s3.go b/pkg/objectstorage/s3.go index 4f43e52593..75ba445ccd 100644 --- a/pkg/objectstorage/s3.go +++ b/pkg/objectstorage/s3.go @@ -16,13 +16,58 @@ package objectstorage import ( "context" + "errors" "io" + "net/http" + "regexp" + "strings" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" ) +// s3Token matches the error codes and request IDs describeS3Error keeps: +// short tokens, as S3's are. +var s3Token = regexp.MustCompile(`^[A-Za-z0-9._=+-]{1,128}$`) + +// describeS3Error describes an S3 response error in err's chain for +// DescribeServiceError by its status, error code and request ID. It leaves out +// the message, which can name the caller and the resource, as AccessDenied's +// does, and a code or request ID that is not a short token. It also leaves out +// the code the SDK makes up when the response has no error body, so such a +// response is described as one. +func describeS3Error(err error) (ServiceError, bool) { + respErr, ok := errors.AsType[s3.ResponseError](err) + if !ok { + return ServiceError{}, false + } + var desc ServiceError + if withStatus, ok := respErr.(interface{ HTTPStatusCode() int }); ok { + desc.HTTPStatus = withStatus.HTTPStatusCode() + } + if id := respErr.ServiceRequestID(); s3Token.MatchString(id) { + desc.RequestID = id + } + apiErr, ok := errors.AsType[smithy.APIError](err) + if ok && !sdkDefaultError(apiErr, desc.HTTPStatus) && s3Token.MatchString(apiErr.ErrorCode()) { + desc.Code = apiErr.ErrorCode() + } + return desc, true +} + +// sdkDefaultError reports whether apiErr carries the code and message the SDK +// fills in for an error response without a body: the status text, or +// "UnknownError" for a status that has none. +func sdkDefaultError(apiErr smithy.APIError, status int) bool { + text := http.StatusText(status) + if text == "" { + text = "UnknownError" + } + return apiErr.ErrorCode() == strings.ReplaceAll(text, " ", "") && apiErr.ErrorMessage() == text +} + const ( // uploadPartSize is how much of a streamed object the uploader buffers before it // sends. A snapshot under this size goes up as a single PutObject; a larger one is diff --git a/pkg/objectstorage/s3_test.go b/pkg/objectstorage/s3_test.go index 395ca71dc3..048f755ca7 100644 --- a/pkg/objectstorage/s3_test.go +++ b/pkg/objectstorage/s3_test.go @@ -17,13 +17,18 @@ package objectstorage import ( "context" "errors" + "fmt" "net/http" "net/http/httptest" + "strings" "testing" "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + smithyhttp "github.com/aws/smithy-go/transport/http" ) // s3ErrorClient starts a test HTTP server returning canned responses so the @@ -97,3 +102,71 @@ func TestS3GetObjectClassifiesAbsence(t *testing.T) { }) } } + +// TestDescribeServiceErrorS3 checks that a failed upload is described by its +// S3 error code and request ID, never by its message or the other elements of +// its body, and that a code or request ID that is not a token, a code the SDK +// made up for a response without a body, and an error from another AWS +// service, such as STS while fetching credentials, are left out. +func TestDescribeServiceErrorS3(t *testing.T) { + const secret = "SECRET-KEY" + stsErr := fmt.Errorf("get identity: %w", &awshttp.ResponseError{ + ResponseError: &smithyhttp.ResponseError{ + Response: &smithyhttp.Response{Response: &http.Response{StatusCode: http.StatusBadRequest}}, + Err: &smithy.GenericAPIError{Code: "ExpiredTokenException", Message: secret}, + }, + RequestID: "sts-req-1", + }) + for _, tc := range []struct { + name string + status int + body string + err error // used instead of an S3 response when set + want ServiceError + }{ + { + name: "error body", + status: http.StatusForbidden, + body: `SignatureDoesNotMatch` + + `Signature for AKIDEXAMPLE does not match ` + secret + `` + + `AKIDEXAMPLE` + secret + `` + + `req-1host-1`, + want: ServiceError{HTTPStatus: http.StatusForbidden, Code: "SignatureDoesNotMatch", RequestID: "req-1"}, + }, + { + name: "code and request ID that are not short tokens", + status: http.StatusBadRequest, + body: `Bad ` + secret + `` + strings.Repeat("a", 129) + ``, + want: ServiceError{HTTPStatus: http.StatusBadRequest}, + }, + { + name: "no error body", + status: http.StatusBadRequest, + want: ServiceError{HTTPStatus: http.StatusBadRequest}, + }, + { + name: "no error body, status without a name", + status: 599, + want: ServiceError{HTTPStatus: 599}, + }, + { + name: "error from another AWS service", + err: stsErr, + want: ServiceError{HTTPStatus: http.StatusBadRequest}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + err := tc.err + if err == nil { + err = s3ErrorClient(t, tc.status, tc.body).PutObject(context.Background(), "bkt", "obj", strings.NewReader("data")) + } + if err == nil { + t.Fatal("PutObject succeeded, want an error") + } + got, ok := DescribeServiceError(err) + if !ok || got != tc.want { + t.Errorf("DescribeServiceError = %+v, %v; want %+v, true", got, ok, tc.want) + } + }) + } +}