diff --git a/api/src/main/java/io/minio/BaseS3Client.java b/api/src/main/java/io/minio/BaseS3Client.java index aaf7c18a8..589d2f618 100644 --- a/api/src/main/java/io/minio/BaseS3Client.java +++ b/api/src/main/java/io/minio/BaseS3Client.java @@ -51,13 +51,11 @@ import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; -import java.security.SecureRandom; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Locale; import java.util.Map; -import java.util.Random; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -92,7 +90,6 @@ public abstract class BaseS3Client implements AutoCloseable { "ServerSideEncryptionConfigurationNotFoundError"; // maximum allowed bucket policy size is 20KiB protected static final int MAX_BUCKET_POLICY_SIZE = 20 * 1024; - protected static final Random RANDOM = new Random(new SecureRandom().nextLong()); protected static final ObjectMapper OBJECT_MAPPER = JsonMapper.builder() .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) @@ -104,6 +101,7 @@ public abstract class BaseS3Client implements AutoCloseable { private static final String UPLOAD_ID = "uploadId"; private static final Set TRACE_QUERY_PARAMS = ImmutableSet.of("retention", "legal-hold", "tagging", UPLOAD_ID, "acl", "attributes"); + private PrintWriter traceStream; protected final Map regionCache = new ConcurrentHashMap<>(); protected String userAgent = Utils.getDefaultUserAgent(); @@ -113,19 +111,49 @@ public abstract class BaseS3Client implements AutoCloseable { protected OkHttpClient httpClient; protected boolean closeHttpClient; + /** + * Maximum attempts per S3 request. Default {@link Retry#MAX_RETRY}. Read on every request via the + * retry interceptor's supplier so runtime tuning via {@link #setMaxRetries(int)} takes immediate + * effect. + */ + protected volatile int maxRetries = Retry.MAX_RETRY; + protected BaseS3Client( Http.BaseUrl baseUrl, Provider provider, OkHttpClient httpClient, boolean closeHttpClient) { this.baseUrl = baseUrl; this.provider = provider; - this.httpClient = httpClient; this.closeHttpClient = closeHttpClient; + this.httpClient = wrapWithRetry(httpClient); } protected BaseS3Client(BaseS3Client client) { this.baseUrl = client.baseUrl; this.provider = client.provider; - this.httpClient = client.httpClient; this.closeHttpClient = client.closeHttpClient; + this.maxRetries = client.maxRetries; + this.httpClient = wrapWithRetry(client.httpClient); + } + + /** + * Re-wires the retry interceptor on {@code client} so it reads {@link #maxRetries} from this + * instance. Strips any prior {@link Http.RetryInterceptor} (e.g. one bound to a different + * instance via the copy constructor) before installing this one. + */ + private OkHttpClient wrapWithRetry(OkHttpClient client) { + OkHttpClient.Builder builder = client.newBuilder().retryOnConnectionFailure(false); + builder.interceptors().removeIf(i -> i instanceof Http.RetryInterceptor); + return builder.addInterceptor(new Http.RetryInterceptor(() -> this.maxRetries)).build(); + } + + /** + * Sets the maximum number of attempts for transient HTTP failures. Pass {@code 1} to disable + * automatic retries. Defaults to {@link Retry#MAX_RETRY}. + * + * @param maxRetries maximum attempts (must be {@code >= 1}). + */ + public void setMaxRetries(int maxRetries) { + if (maxRetries < 1) throw new IllegalArgumentException("maxRetries must be >= 1"); + this.maxRetries = maxRetries; } /** Closes underneath HTTP client. */ @@ -268,7 +296,13 @@ private String[] handleRedirectResponse( return new String[] {code, message}; } - /** Execute HTTP request asynchronously for given parameters. */ + /** + * Execute HTTP request asynchronously for given parameters. Retries on retryable IOException, + * HTTP status, and S3 error code are handled by {@link Http.RetryInterceptor}, which {@link + * #wrapWithRetry} installs on the underlying {@link OkHttpClient} regardless of whether the + * client is the default or caller-supplied. The attempt budget is read from {@link #maxRetries} + * on every request. + */ protected CompletableFuture executeAsync(Http.S3Request s3request, String region) { Credentials credentials = (provider == null) ? null : provider.fetch(); Http.Request request = null; @@ -282,15 +316,9 @@ protected CompletableFuture executeAsync(Http.S3Request s3request, Str PrintWriter traceStream = this.traceStream; if (traceStream != null) traceStream.print(request.httpTraces()); - OkHttpClient httpClient = this.httpClient; - // FIXME: enable retry for all request. - // if (!s3request.retryFailure()) { - // httpClient = httpClient.newBuilder().retryOnConnectionFailure(false).build(); - // } - okhttp3.Request httpRequest = request.httpRequest(); CompletableFuture completableFuture = newCompleteableFuture(); - httpClient + this.httpClient .newCall(httpRequest) .enqueue( new Callback() { @@ -469,9 +497,10 @@ private void onResponse(final Response response) throws IOException { response.header("x-amz-id-2")); } - // invalidate region cache if needed - if (errorResponse.code().equals(NO_SUCH_BUCKET) - || errorResponse.code().equals(RETRY_HEAD)) { + // invalidate region cache if needed (bucket may be null for e.g. listBuckets) + if (s3request.bucket() != null + && (errorResponse.code().equals(NO_SUCH_BUCKET) + || errorResponse.code().equals(RETRY_HEAD))) { regionCache.remove(s3request.bucket()); } @@ -1223,6 +1252,15 @@ private Object[] createBody(PutObjectAPIBaseArgs args, MediaType contentType) boolean checksumHeader = headers.namePrefixAny("x-amz-checksum-"); String md5Hash = headers.getFirst(Http.Headers.CONTENT_MD5); + long fileStartPos = 0; + if (args.file() != null) { + try { + fileStartPos = args.file().getFilePointer(); + } catch (IOException e) { + throw new MinioException(e); + } + } + if (sha256HexString == null && sha256Base64String == null) { if (!baseUrl.isHttps()) { Checksum.Hasher hasher = Checksum.Algorithm.SHA256.hasher(); @@ -1278,6 +1316,14 @@ private Object[] createBody(PutObjectAPIBaseArgs args, MediaType contentType) } } + if (args.file() != null) { + try { + args.file().seek(fileStartPos); + } catch (IOException e) { + throw new MinioException(e); + } + } + Http.Body body = null; if (args.file() != null) { body = new Http.Body(args.file(), args.length(), contentType, sha256HexString, md5Hash); diff --git a/api/src/main/java/io/minio/Http.java b/api/src/main/java/io/minio/Http.java index 846bfc3b9..0d050ac8d 100644 --- a/api/src/main/java/io/minio/Http.java +++ b/api/src/main/java/io/minio/Http.java @@ -53,6 +53,7 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.function.IntSupplier; import java.util.regex.Matcher; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -65,6 +66,7 @@ import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509TrustManager; +import okhttp3.Interceptor; import okhttp3.MediaType; import okhttp3.OkHttpClient; import okhttp3.Protocol; @@ -620,6 +622,10 @@ public static OkHttpClient newDefaultClient() { .writeTimeout(DEFAULT_TIMEOUT, TimeUnit.MILLISECONDS) .readTimeout(DEFAULT_TIMEOUT, TimeUnit.MILLISECONDS) .protocols(Arrays.asList(Protocol.HTTP_1_1)) + // Our RetryInterceptor handles transient failures with full-jitter backoff; defer + // entirely to it instead of layering OkHttp's connection-level retry on top. + .retryOnConnectionFailure(false) + .addInterceptor(new RetryInterceptor()) .build(); try { return enableExternalCertificatesFromEnv(client); @@ -683,10 +689,131 @@ public static OkHttpClient setTimeout( .build(); } + /** + * OkHttp interceptor that retries transient HTTP failures with full-jitter exponential backoff. + * + *

This is part of the SDK's supported public API: it is installed automatically by {@link + * BaseS3Client} on every supplied {@link OkHttpClient} (default or caller-provided), and may also + * be registered explicitly on a stand-alone {@link OkHttpClient} via {@code .addInterceptor(new + * Http.RetryInterceptor())}. + * + *

Retries on: + * + *

    + *
  • retryable IOException — connection reset, EOF, socket timeout, idle-connection close. + * Excludes TLS handshake / unknown-CA / HTTPS protocol mismatch. + *
  • retryable HTTP status code — 408, 429, 499, 500, 502, 503, 504, 520. + *
  • retryable S3 error code in a non-2xx response body — {@code SlowDown}, {@code + * InternalError}, {@code ExpiredToken}, etc. + *
+ * + *

Backoff is full-jitter exponential, with a 200 ms unit and a 1 s per-attempt cap. + * The maximum number of attempts is supplied per intercept call via {@link IntSupplier} so that + * an SDK client can expose runtime tuning while keeping the interceptor itself stateless. The + * no-arg constructor uses the package default of 10 attempts. + * + *

Threading. Backoff sleeps on the OkHttp dispatcher thread that owns the call. Under + * sustained 5xx/429 storms this can hold dispatcher slots idle while waiting. Callers that need + * higher concurrency under widespread retry should size the dispatcher worker pool accordingly + * (see {@link okhttp3.Dispatcher#setMaxRequests} / {@code setMaxRequestsPerHost}). + * + *

To opt out of retries on a stand-alone {@link OkHttpClient}, simply do not register this + * interceptor. + */ + public static class RetryInterceptor implements Interceptor { + /** Maximum body bytes inspected when probing for an S3 {@code } value. */ + private static final long MAX_PEEK_BYTES = 5L * 1024L * 1024L; + + private static final java.util.regex.Pattern S3_ERROR_CODE_PATTERN = + java.util.regex.Pattern.compile("([^<]+)"); + + private final IntSupplier maxAttemptsSupplier; + + /** Creates a retry interceptor that uses {@link Retry#MAX_RETRY} attempts. */ + public RetryInterceptor() { + this(() -> Retry.MAX_RETRY); + } + + /** + * Creates a retry interceptor that reads its attempt budget from the supplier on every + * intercept call. Supplier values less than 1 are clamped to 1 (single attempt = retry off). + */ + public RetryInterceptor(IntSupplier maxAttemptsSupplier) { + this.maxAttemptsSupplier = Objects.requireNonNull(maxAttemptsSupplier, "maxAttemptsSupplier"); + } + + @Override + public okhttp3.Response intercept(Chain chain) throws IOException { + okhttp3.Request request = chain.request(); + int maxAttempts = Math.max(1, maxAttemptsSupplier.getAsInt()); + + okhttp3.Response response = null; + IOException lastException = null; + + for (int attempt = 0; attempt < maxAttempts; attempt++) { + if (attempt > 0) { + long delayMs = Retry.exponentialBackoffMs(attempt - 1); + if (delayMs > 0L) { + try { + Thread.sleep(delayMs); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new IOException("Retry interrupted", ie); + } + } + } + + if (response != null) { + response.close(); + response = null; + } + + try { + response = chain.proceed(request); + } catch (IOException e) { + if (!Retry.isRequestErrorRetryable(e)) throw e; + lastException = e; + continue; + } + + if (Retry.isHttpStatusRetryable(response.code())) { + lastException = null; + continue; + } + + if (!response.isSuccessful()) { + String s3Code = peekS3ErrorCode(response); + if (Retry.isS3CodeRetryable(s3Code)) { + lastException = null; + continue; + } + } + + return response; + } + + if (lastException != null) throw lastException; + return response; + } + + /** Returns the S3 {@code } value if the body is XML containing one, else null. */ + private static String peekS3ErrorCode(okhttp3.Response response) { + try { + String body = response.peekBody(MAX_PEEK_BYTES).string(); + if (body.isEmpty() || !body.contains("= 1"); + this.maxRetries = maxRetries; + return this; + } + public MinioAsyncClient build() { Utils.validateNotNull(baseUrl, "endpoint"); @@ -232,7 +244,10 @@ public MinioAsyncClient build() { httpClient = Http.newDefaultClient(); } - return new MinioAsyncClient(baseUrl, provider, httpClient, closeHttpClient); + MinioAsyncClient client = + new MinioAsyncClient(baseUrl, provider, httpClient, closeHttpClient); + client.maxRetries = maxRetries; + return client; } } @@ -3356,7 +3371,7 @@ public CompletableFuture putObjectFanOut(PutObjectFanOu // Build POST object data String objectName = "fan-out-" - + new BigInteger(32, RANDOM).toString(32) + + new BigInteger(32, ThreadLocalRandom.current()).toString(32) + "-" + System.currentTimeMillis(); PostPolicy policy = diff --git a/api/src/main/java/io/minio/MinioClient.java b/api/src/main/java/io/minio/MinioClient.java index 03dd587a9..839a14cbd 100644 --- a/api/src/main/java/io/minio/MinioClient.java +++ b/api/src/main/java/io/minio/MinioClient.java @@ -1921,6 +1921,16 @@ public void ignoreCertCheck() throws MinioException { asyncClient.ignoreCertCheck(); } + /** + * Sets the maximum number of attempts for transient HTTP failures. Pass {@code 1} to disable + * automatic retries. Defaults to 10. + * + * @param maxRetries maximum attempts (must be {@code >= 1}). + */ + public void setMaxRetries(int maxRetries) { + asyncClient.setMaxRetries(maxRetries); + } + /** * Sets application's name/version to user agent. For more information about user agent refer #rfc2616. @@ -2040,6 +2050,15 @@ public Builder httpClient(OkHttpClient httpClient, boolean close) { return this; } + /** + * Sets the maximum number of attempts per request. Pass {@code 1} to disable automatic retries. + * Defaults to 10. + */ + public Builder maxRetries(int maxRetries) { + asyncClientBuilder.maxRetries(maxRetries); + return this; + } + public MinioClient build() { MinioAsyncClient asyncClient = asyncClientBuilder.build(); return new MinioClient(asyncClient); diff --git a/api/src/main/java/io/minio/Retry.java b/api/src/main/java/io/minio/Retry.java new file mode 100644 index 000000000..72c06d6c2 --- /dev/null +++ b/api/src/main/java/io/minio/Retry.java @@ -0,0 +1,128 @@ +/* + * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2026 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.minio; + +import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.security.cert.CertPathBuilderException; +import java.security.cert.CertPathValidatorException; +import java.security.cert.CertificateException; +import java.util.Set; +import java.util.concurrent.ThreadLocalRandom; +import javax.net.ssl.SSLException; +import javax.net.ssl.SSLHandshakeException; +import javax.net.ssl.SSLPeerUnverifiedException; + +/** + * Retry configuration and classification helpers used by {@link Http.RetryInterceptor}. + * + *

Defines the retryable HTTP status set, retryable S3 error code set, IOException filter, and + * the full-jitter exponential backoff formula used for transient failure recovery. + */ +class Retry { + /** Default maximum number of attempts per request. */ + static final int MAX_RETRY = 10; + + /** Base unit per retry attempt, in milliseconds. */ + static final long DEFAULT_RETRY_UNIT_MS = 200L; + + /** Per-attempt sleep cap, in milliseconds. */ + static final long DEFAULT_RETRY_CAP_MS = 1_000L; + + /** Maximum jitter fraction in {@code [0.0, 1.0]}. {@code 1.0} = full jitter. */ + static final double MAX_JITTER = 1.0; + + /** Retryable AWS S3 error codes. */ + static final Set RETRYABLE_S3_CODES = + ImmutableSet.of( + "RequestError", + "RequestTimeout", + "Throttling", + "ThrottlingException", + "RequestLimitExceeded", + "RequestThrottled", + "InternalError", + "ExpiredToken", + "ExpiredTokenException", + "SlowDown", + "SlowDownWrite", + "SlowDownRead"); + + /** Retryable HTTP status codes. */ + static final Set RETRYABLE_HTTP_STATUS_CODES = + ImmutableSet.of( + 408, // Request Timeout + 429, // Too Many Requests + 499, // Client Closed Request (nginx) + 500, // Internal Server Error + 502, // Bad Gateway + 503, // Service Unavailable + 504, // Gateway Timeout + 520); // Cloudflare unknown error + + static boolean isS3CodeRetryable(String code) { + return code != null && RETRYABLE_S3_CODES.contains(code); + } + + static boolean isHttpStatusRetryable(int code) { + return RETRYABLE_HTTP_STATUS_CODES.contains(code); + } + + /** + * Returns true if {@code e} represents a transient transport failure that should be retried. TLS + * handshake failure, unknown-CA / cert-path errors, and the "server gave HTTP response to HTTPS + * client" protocol mismatch are NOT retryable; everything else (connection reset, EOF, server + * closed idle connection, socket timeout, …) is. + */ + static boolean isRequestErrorRetryable(IOException e) { + if (e instanceof SSLHandshakeException) return false; + if (e instanceof SSLPeerUnverifiedException) return false; + if (e instanceof SSLException) { + Throwable cause = e.getCause(); + if (cause instanceof CertPathBuilderException + || cause instanceof CertPathValidatorException + || cause instanceof CertificateException) { + return false; + } + } + String msg = e.getMessage(); + if (msg != null && msg.contains("server gave HTTP response to HTTPS client")) return false; + return true; + } + + /** + * Computes the exponential-backoff-with-full-jitter delay for retry {@code attempt} (0-indexed: + * {@code 0} = before the second attempt, {@code 1} = before the third, …): + * + * 

+   *   sleep = min(DEFAULT_RETRY_CAP_MS, DEFAULT_RETRY_UNIT_MS * 2^attempt)
+   *   sleep -= random.nextDouble() * sleep * MAX_JITTER     // full jitter when MAX_JITTER == 1.0
+   *
+ * + *

With {@code MAX_JITTER == 1.0}, returns a uniform random value in {@code [0, min(cap, base * + * 2^attempt)]}. + */ + static long exponentialBackoffMs(int attempt) { + int exp = Math.min(Math.max(attempt, 0), 30); + long sleep = DEFAULT_RETRY_UNIT_MS * (1L << exp); + if (sleep > DEFAULT_RETRY_CAP_MS) sleep = DEFAULT_RETRY_CAP_MS; + sleep -= (long) (ThreadLocalRandom.current().nextDouble() * (double) sleep * MAX_JITTER); + return Math.max(0L, sleep); + } + + private Retry() {} +} diff --git a/api/src/test/java/io/minio/RetryTest.java b/api/src/test/java/io/minio/RetryTest.java new file mode 100644 index 000000000..0f3fcf1e0 --- /dev/null +++ b/api/src/test/java/io/minio/RetryTest.java @@ -0,0 +1,409 @@ +/* + * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2026 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.minio; + +import io.minio.errors.ErrorResponseException; +import io.minio.errors.InvalidResponseException; +import io.minio.errors.MinioException; +import java.io.IOException; +import java.net.SocketException; +import javax.net.ssl.SSLHandshakeException; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okio.Buffer; +import org.junit.Assert; +import org.junit.Test; + +/** Unit + integration tests for {@link Retry} and {@link Http.RetryInterceptor}. */ +public class RetryTest { + + // --------------------------------------------------------------------------- + // Retry.isHttpStatusRetryable + // --------------------------------------------------------------------------- + + @Test + public void testIsHttpStatusRetryable_retryable() { + Assert.assertTrue(Retry.isHttpStatusRetryable(408)); + Assert.assertTrue(Retry.isHttpStatusRetryable(429)); + Assert.assertTrue(Retry.isHttpStatusRetryable(499)); + Assert.assertTrue(Retry.isHttpStatusRetryable(500)); + Assert.assertTrue(Retry.isHttpStatusRetryable(502)); + Assert.assertTrue(Retry.isHttpStatusRetryable(503)); + Assert.assertTrue(Retry.isHttpStatusRetryable(504)); + Assert.assertTrue(Retry.isHttpStatusRetryable(520)); + } + + @Test + public void testIsHttpStatusRetryable_notRetryable() { + for (int code : new int[] {200, 201, 204, 301, 304, 400, 401, 403, 404, 409, 412, 416, 501}) { + Assert.assertFalse( + "status " + code + " must not be retryable", Retry.isHttpStatusRetryable(code)); + } + } + + // --------------------------------------------------------------------------- + // Retry.isS3CodeRetryable + // --------------------------------------------------------------------------- + + @Test + public void testIsS3CodeRetryable_retryable() { + String[] codes = { + "RequestError", + "RequestTimeout", + "Throttling", + "ThrottlingException", + "RequestLimitExceeded", + "RequestThrottled", + "InternalError", + "ExpiredToken", + "ExpiredTokenException", + "SlowDown", + "SlowDownWrite", + "SlowDownRead" + }; + for (String c : codes) { + Assert.assertTrue("S3 code " + c + " must be retryable", Retry.isS3CodeRetryable(c)); + } + } + + @Test + public void testIsS3CodeRetryable_notRetryable() { + Assert.assertFalse(Retry.isS3CodeRetryable("NoSuchKey")); + Assert.assertFalse(Retry.isS3CodeRetryable("NoSuchBucket")); + Assert.assertFalse(Retry.isS3CodeRetryable("AccessDenied")); + Assert.assertFalse(Retry.isS3CodeRetryable("RetryHead")); + Assert.assertFalse(Retry.isS3CodeRetryable("")); + Assert.assertFalse(Retry.isS3CodeRetryable(null)); + } + + // --------------------------------------------------------------------------- + // Retry.isRequestErrorRetryable + // --------------------------------------------------------------------------- + + @Test + public void testIsRequestErrorRetryable_retryable() { + Assert.assertTrue(Retry.isRequestErrorRetryable(new IOException("connection reset"))); + Assert.assertTrue(Retry.isRequestErrorRetryable(new IOException("EOF"))); + Assert.assertTrue( + Retry.isRequestErrorRetryable(new IOException("http: server closed idle connection"))); + Assert.assertTrue(Retry.isRequestErrorRetryable(new SocketException("Connection timed out"))); + Assert.assertTrue(Retry.isRequestErrorRetryable(new java.net.SocketTimeoutException("read"))); + } + + @Test + public void testIsRequestErrorRetryable_sslHandshakeNotRetryable() { + Assert.assertFalse( + Retry.isRequestErrorRetryable(new SSLHandshakeException("cert not trusted"))); + } + + @Test + public void testIsRequestErrorRetryable_protocolMismatchNotRetryable() { + Assert.assertFalse( + Retry.isRequestErrorRetryable( + new IOException("server gave HTTP response to HTTPS client"))); + } + + // --------------------------------------------------------------------------- + // Retry.exponentialBackoffMs + // --------------------------------------------------------------------------- + + @Test + public void testExponentialBackoffMs_attempt0WithinFirstUnit() { + // attempt=0: cap = min(1000, 200*2^0) = 200ms; with full jitter, result in [0, 200]. + for (int i = 0; i < 100; i++) { + long delay = Retry.exponentialBackoffMs(0); + Assert.assertTrue( + "attempt 0 delay must be in [0, 200ms], got " + delay, delay >= 0 && delay <= 200); + } + } + + @Test + public void testExponentialBackoffMs_attempt2WithinSecondCap() { + // attempt=2: cap = min(1000, 200*2^2) = 800ms. + for (int i = 0; i < 100; i++) { + long delay = Retry.exponentialBackoffMs(2); + Assert.assertTrue( + "attempt 2 delay must be in [0, 800ms], got " + delay, delay >= 0 && delay <= 800); + } + } + + @Test + public void testExponentialBackoffMs_cappedAtRetryCap() { + // attempt=10: uncapped 200*2^10 = 204800ms; capped at 1000ms. + for (int i = 0; i < 100; i++) { + long delay = Retry.exponentialBackoffMs(10); + Assert.assertTrue("delay must be <= cap, got " + delay, delay <= Retry.DEFAULT_RETRY_CAP_MS); + Assert.assertTrue(delay >= 0); + } + } + + @Test + public void testExponentialBackoffMs_negativeAttemptIsClamped() { + // Negative attempt clamps to 0; cap = 200ms. + long delay = Retry.exponentialBackoffMs(-5); + Assert.assertTrue(delay >= 0 && delay <= 200); + } + + @Test + public void testExponentialBackoffMs_highAttemptDoesNotOverflow() { + // High attempts must not bit-shift overflow; cap saturates. + for (int attempt : new int[] {30, 31, 60, 100, 1000}) { + long delay = Retry.exponentialBackoffMs(attempt); + Assert.assertTrue( + "attempt=" + attempt + " delay must be <= cap, got " + delay, + delay <= Retry.DEFAULT_RETRY_CAP_MS); + Assert.assertTrue("attempt=" + attempt + " delay must be >= 0", delay >= 0); + } + } + + // --------------------------------------------------------------------------- + // Integration tests via MockWebServer + // --------------------------------------------------------------------------- + + private static final String LIST_BUCKETS_OK = + "" + + "" + + "testtest" + + ""; + + private MockResponse successResponse() { + return new MockResponse() + .setResponseCode(200) + .setHeader("Content-Type", "application/xml") + .setBody(new Buffer().writeUtf8(LIST_BUCKETS_OK)); + } + + private MockResponse htmlServerError(int code) { + return new MockResponse() + .setResponseCode(code) + .setHeader("Content-Type", "text/html") + .setBody(new Buffer().writeUtf8("" + code + "")); + } + + private MockResponse xmlError(int code, String s3Code) { + String xml = + "" + + "" + + s3Code + + "m/id"; + return new MockResponse() + .setResponseCode(code) + .setHeader("Content-Type", "application/xml") + .setBody(new Buffer().writeUtf8(xml)); + } + + @Test + public void testRetryOn503ThenSuccess() throws IOException, MinioException { + try (MockWebServer server = new MockWebServer()) { + server.enqueue(htmlServerError(503)); + server.enqueue(successResponse()); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(2).build(); + client.listBuckets(); + + Assert.assertEquals(2, server.getRequestCount()); + } + } + + @Test + public void testRetryOnEachRetryableHttpCode() throws IOException, MinioException { + for (int code : new int[] {408, 429, 499, 500, 502, 503, 504, 520}) { + try (MockWebServer server = new MockWebServer()) { + server.enqueue(htmlServerError(code)); + server.enqueue(successResponse()); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(2).build(); + client.listBuckets(); + + Assert.assertEquals("status " + code + " expected to retry", 2, server.getRequestCount()); + } + } + } + + @Test + public void testRetryOnRetryableS3CodeIn400Body() throws IOException, MinioException { + // 400 is NOT a retryable HTTP status, but body has retryable S3 code → retry. + try (MockWebServer server = new MockWebServer()) { + server.enqueue(xmlError(400, "ExpiredToken")); + server.enqueue(successResponse()); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(2).build(); + client.listBuckets(); + + Assert.assertEquals(2, server.getRequestCount()); + } + } + + @Test + public void testNoRetryOn404() throws IOException, MinioException { + try (MockWebServer server = new MockWebServer()) { + server.enqueue(xmlError(404, "NoSuchBucket")); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(3).build(); + try { + client.listBuckets(); + Assert.fail("expected ErrorResponseException"); + } catch (ErrorResponseException e) { + Assert.assertEquals("NoSuchBucket", e.errorResponse().code()); + Assert.assertEquals(404, e.response().code()); + } + Assert.assertEquals(1, server.getRequestCount()); + } + } + + @Test + public void testNoRetryOn403() throws IOException, MinioException { + try (MockWebServer server = new MockWebServer()) { + server.enqueue(xmlError(403, "AccessDenied")); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(3).build(); + try { + client.listBuckets(); + Assert.fail("expected ErrorResponseException"); + } catch (ErrorResponseException e) { + Assert.assertEquals("AccessDenied", e.errorResponse().code()); + } + Assert.assertEquals(1, server.getRequestCount()); + } + } + + @Test + public void testRetryExhaustedReturnsLastResponse() throws IOException, MinioException { + // 3 attempts, all 500 (HTML so server-side response dispatches to InvalidResponseException). + try (MockWebServer server = new MockWebServer()) { + server.enqueue(htmlServerError(500)); + server.enqueue(htmlServerError(500)); + server.enqueue(htmlServerError(500)); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(3).build(); + try { + client.listBuckets(); + Assert.fail("expected exception after exhausted retries"); + } catch (InvalidResponseException e) { + // Terminal exception must surface the underlying 500 status so callers can distinguish + // exhausted retries from unrelated failures (e.g. NPE, parser bugs). + Assert.assertTrue( + "exhausted-retries exception must reflect HTTP 500, got: " + e.getMessage(), + e.getMessage().contains("Response code: 500")); + } + Assert.assertEquals(3, server.getRequestCount()); + } + } + + @Test + public void testRetryExhaustedSurfacesXmlErrorResponse() throws IOException, MinioException { + // 3 attempts, all 503 with XML InternalError — confirms the terminal + // ErrorResponseException carries both the 503 status and the parsed S3 code after the retry + // loop gives up. + try (MockWebServer server = new MockWebServer()) { + server.enqueue(xmlError(503, "InternalError")); + server.enqueue(xmlError(503, "InternalError")); + server.enqueue(xmlError(503, "InternalError")); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(3).build(); + try { + client.listBuckets(); + Assert.fail("expected ErrorResponseException after exhausted retries"); + } catch (ErrorResponseException e) { + Assert.assertEquals(503, e.response().code()); + Assert.assertEquals("InternalError", e.errorResponse().code()); + } + Assert.assertEquals(3, server.getRequestCount()); + } + } + + @Test + public void testMaxRetriesOneDisablesRetry() throws IOException, MinioException { + try (MockWebServer server = new MockWebServer()) { + server.enqueue(htmlServerError(503)); + // Second response should never be reached. + server.enqueue(successResponse()); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(1).build(); + try { + client.listBuckets(); + Assert.fail("expected exception"); + } catch (InvalidResponseException e) { + // expected + } + Assert.assertEquals(1, server.getRequestCount()); + } + } + + @Test + public void testSetMaxRetriesPostConstruction() throws IOException, MinioException { + try (MockWebServer server = new MockWebServer()) { + server.enqueue(htmlServerError(503)); + server.enqueue(successResponse()); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(2).build(); + client.setMaxRetries(1); // disable + try { + client.listBuckets(); + Assert.fail("expected exception"); + } catch (InvalidResponseException e) { + // expected + } + Assert.assertEquals(1, server.getRequestCount()); + } + } + + @Test(expected = IllegalArgumentException.class) + public void testMaxRetriesBuilderValidation() { + MinioClient.builder().endpoint("http://localhost:9000").maxRetries(0).build(); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetMaxRetriesValidation() { + MinioClient client = MinioClient.builder().endpoint("http://localhost:9000").build(); + client.setMaxRetries(0); + } + + @Test + public void testMultipleRetrySucceedsOnThirdAttempt() throws IOException, MinioException { + try (MockWebServer server = new MockWebServer()) { + server.enqueue(htmlServerError(500)); + server.enqueue(htmlServerError(503)); + server.enqueue(successResponse()); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(3).build(); + client.listBuckets(); + + Assert.assertEquals(3, server.getRequestCount()); + } + } +}