From 767663459485f2f1a10ccebbf7dc6ebd8230b73e Mon Sep 17 00:00:00 2001 From: Allan Roger Reid Date: Mon, 27 Apr 2026 13:01:43 +0000 Subject: [PATCH 01/11] fix: retry transient per-object delete failures in cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The removeObjects() cleanup helper silently ignored all DeleteErrors via ignore(r.get()), masking transient server-side failures. When the batch delete of 1050 objects returned partial errors (e.g. SlowDown, InternalError), those objects remained in the bucket and the subsequent removeBucket() call failed with BucketNotEmpty — surfacing a cleanup race as a false test failure. Replace the single ignore() call with two focused helpers: - retryRemoveObject(): retries a single object with exponential backoff (500ms base) up to MAX_DELETE_RETRIES attempts. Accepts NoSuchKey/NoSuchVersion silently (already gone), retries SlowDown/InternalError/RequestTimeout/ServiceUnavailable, and propagates anything else immediately. - removeObjects(): fully consumes the batch-delete iterator first (so all 1000-object HTTP batches are sent), then delegates each DeleteResult.Error to retryRemoveObject() individually. DeleteResult.Error only exposes objectName() via ErrorResponse (no versionId), so a HashMap> index preserves correct versioned-delete semantics by storing all responses keyed by name and retrying every (name,versionId) pair on a miss. --- functional/TestMinioClient.java | 45 +++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/functional/TestMinioClient.java b/functional/TestMinioClient.java index 4bf820d4f..572e0d205 100644 --- a/functional/TestMinioClient.java +++ b/functional/TestMinioClient.java @@ -92,6 +92,7 @@ import io.minio.messages.AccessControlPolicy; import io.minio.messages.CORSConfiguration; import io.minio.messages.DeleteRequest; +import io.minio.messages.DeleteResult; import io.minio.messages.ErrorResponse; import io.minio.messages.EventType; import io.minio.messages.Filter; @@ -121,8 +122,10 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import okhttp3.Headers; @@ -137,6 +140,12 @@ value = "REC", justification = "Allow catching super class Exception since it's tests") public class TestMinioClient extends TestArgs { + private static final int MAX_DELETE_RETRIES = 5; + private static final Set IGNORABLE_DELETE_CODES = + new HashSet<>(Arrays.asList("NoSuchKey", "NoSuchVersion")); + private static final Set TRANSIENT_DELETE_CODES = + new HashSet<>(Arrays.asList("InternalError", "RequestTimeout", "ServiceUnavailable", "SlowDown")); + private String bucketName = getRandomName(); private String bucketNameWithLock = getRandomName(); public boolean isQuickTest; @@ -1027,18 +1036,44 @@ public List createObjects(String bucketName, int count, int return results; } + private void retryRemoveObject(String bucket, String object, String versionId) throws Exception { + for (int attempt = 0; attempt < MAX_DELETE_RETRIES; attempt++) { + if (attempt > 0) Thread.sleep(500L << attempt); + try { + RemoveObjectArgs.Builder b = RemoveObjectArgs.builder().bucket(bucket).object(object); + if (versionId != null) b.versionId(versionId); + client.removeObject(b.build()); + return; + } catch (ErrorResponseException e) { + String code = e.errorResponse().code(); + if (IGNORABLE_DELETE_CODES.contains(code)) return; + if (attempt == MAX_DELETE_RETRIES - 1 || !TRANSIENT_DELETE_CODES.contains(code)) throw e; + } + } + } + public void removeObjects(String bucketName, List results) throws Exception { + // DeleteResult.Error only exposes objectName(); index all responses for retry lookup. + Map> byName = new HashMap<>(); + for (ObjectWriteResponse res : results) { + byName.computeIfAbsent(res.object(), k -> new ArrayList<>()).add(res); + } List objects = results.stream() - .map( - result -> { - return new DeleteRequest.Object(result.object(), result.versionId()); - }) + .map(result -> new DeleteRequest.Object(result.object(), result.versionId())) .collect(Collectors.toList()); + // Fully consume the iterator before retrying so all batches are sent first. + List deleteErrors = new ArrayList<>(); for (Result r : client.removeObjects( RemoveObjectsArgs.builder().bucket(bucketName).objects(objects).build())) { - ignore(r.get()); + DeleteResult.Error err = (DeleteResult.Error) r.get(); + if (err != null) deleteErrors.add(err); + } + for (DeleteResult.Error err : deleteErrors) { + for (ObjectWriteResponse res : byName.getOrDefault(err.objectName(), new ArrayList<>())) { + retryRemoveObject(bucketName, res.object(), res.versionId()); + } } } From 30b5c350210425d720aa56f38a11d3e8d8b777ee Mon Sep 17 00:00:00 2001 From: Allan Roger Reid Date: Mon, 27 Apr 2026 14:34:35 +0000 Subject: [PATCH 02/11] style: apply Google Java Format (spotless) --- functional/TestMinioClient.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/functional/TestMinioClient.java b/functional/TestMinioClient.java index 572e0d205..01c052572 100644 --- a/functional/TestMinioClient.java +++ b/functional/TestMinioClient.java @@ -144,7 +144,8 @@ public class TestMinioClient extends TestArgs { private static final Set IGNORABLE_DELETE_CODES = new HashSet<>(Arrays.asList("NoSuchKey", "NoSuchVersion")); private static final Set TRANSIENT_DELETE_CODES = - new HashSet<>(Arrays.asList("InternalError", "RequestTimeout", "ServiceUnavailable", "SlowDown")); + new HashSet<>( + Arrays.asList("InternalError", "RequestTimeout", "ServiceUnavailable", "SlowDown")); private String bucketName = getRandomName(); private String bucketNameWithLock = getRandomName(); From 14c716b1b89167e5c75442a5d6bca37a74b4613b Mon Sep 17 00:00:00 2001 From: Allan Roger Reid Date: Mon, 27 Apr 2026 18:57:49 +0000 Subject: [PATCH 03/11] fix: tighten removeObjects retry loop type-safety and resource handling Switch Result to Result to restore compile-time type safety and eliminate the unchecked cast. Drain the full server response before throwing on a non-transient error so OkHttp can reuse the connection slot. Restore the thread interrupt flag if Thread.sleep is interrupted. Wrap TRANSIENT_DELETE_CODES in an unmodifiable set. Guard the testRemoveObjects finally block so the redundant cleanup batch is only sent on failure. Add bucket name to error messages and document the retry-by-name re-queueing trade-off. --- functional/TestMinioClient.java | 92 ++++++++++++++++++++------------- 1 file changed, 57 insertions(+), 35 deletions(-) diff --git a/functional/TestMinioClient.java b/functional/TestMinioClient.java index 01c052572..7e95903f4 100644 --- a/functional/TestMinioClient.java +++ b/functional/TestMinioClient.java @@ -113,6 +113,7 @@ import io.minio.messages.Tags; import io.minio.messages.VersioningConfiguration; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -141,11 +142,10 @@ justification = "Allow catching super class Exception since it's tests") public class TestMinioClient extends TestArgs { private static final int MAX_DELETE_RETRIES = 5; - private static final Set IGNORABLE_DELETE_CODES = - new HashSet<>(Arrays.asList("NoSuchKey", "NoSuchVersion")); private static final Set TRANSIENT_DELETE_CODES = - new HashSet<>( - Arrays.asList("InternalError", "RequestTimeout", "ServiceUnavailable", "SlowDown")); + Collections.unmodifiableSet( + new HashSet<>( + Arrays.asList("InternalError", "RequestTimeout", "ServiceUnavailable", "SlowDown"))); private String bucketName = getRandomName(); private String bucketNameWithLock = getRandomName(); @@ -1037,44 +1037,64 @@ public List createObjects(String bucketName, int count, int return results; } - private void retryRemoveObject(String bucket, String object, String versionId) throws Exception { - for (int attempt = 0; attempt < MAX_DELETE_RETRIES; attempt++) { - if (attempt > 0) Thread.sleep(500L << attempt); - try { - RemoveObjectArgs.Builder b = RemoveObjectArgs.builder().bucket(bucket).object(object); - if (versionId != null) b.versionId(versionId); - client.removeObject(b.build()); - return; - } catch (ErrorResponseException e) { - String code = e.errorResponse().code(); - if (IGNORABLE_DELETE_CODES.contains(code)) return; - if (attempt == MAX_DELETE_RETRIES - 1 || !TRANSIENT_DELETE_CODES.contains(code)) throw e; - } - } - } - public void removeObjects(String bucketName, List results) throws Exception { - // DeleteResult.Error only exposes objectName(); index all responses for retry lookup. + // DeleteResult.Error has no versionId; keyed by name to rebuild versioned retry batches. Map> byName = new HashMap<>(); for (ObjectWriteResponse res : results) { byName.computeIfAbsent(res.object(), k -> new ArrayList<>()).add(res); } - List objects = + List toDelete = results.stream() - .map(result -> new DeleteRequest.Object(result.object(), result.versionId())) + .map(r -> new DeleteRequest.Object(r.object(), r.versionId())) .collect(Collectors.toList()); - // Fully consume the iterator before retrying so all batches are sent first. - List deleteErrors = new ArrayList<>(); - for (Result r : - client.removeObjects( - RemoveObjectsArgs.builder().bucket(bucketName).objects(objects).build())) { - DeleteResult.Error err = (DeleteResult.Error) r.get(); - if (err != null) deleteErrors.add(err); - } - for (DeleteResult.Error err : deleteErrors) { - for (ObjectWriteResponse res : byName.getOrDefault(err.objectName(), new ArrayList<>())) { - retryRemoveObject(bucketName, res.object(), res.versionId()); + for (int attempt = 0; attempt < MAX_DELETE_RETRIES && !toDelete.isEmpty(); attempt++) { + if (attempt > 0) { + try { + Thread.sleep(500L << attempt); // 1s / 2s / 4s / 8s + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw ie; + } } + Set retryNames = new HashSet<>(); + IOException nonTransientErr = null; + for (Result r : + client.removeObjects( + RemoveObjectsArgs.builder().bucket(bucketName).objects(toDelete).build())) { + DeleteResult.Error err = r.get(); + String code = err.code(); + if (!TRANSIENT_DELETE_CODES.contains(code)) { + if (nonTransientErr == null) { + nonTransientErr = + new IOException( + "non-transient delete error '" + + code + + "' on " + + err.objectName() + + " in bucket " + + bucketName); + } + continue; // drain remaining response before throwing + } + retryNames.add(err.objectName()); + } + if (nonTransientErr != null) throw nonTransientErr; + // All versions re-queued because DeleteResult.Error lacks versionId; + // already-deleted versions return NoSuchVersion, filtered upstream by MinioAsyncClient. + toDelete = new ArrayList<>(); + for (String name : retryNames) { + for (ObjectWriteResponse res : byName.getOrDefault(name, Collections.emptyList())) { + toDelete.add(new DeleteRequest.Object(res.object(), res.versionId())); + } + } + } + if (!toDelete.isEmpty()) { + throw new IOException( + toDelete.size() + + " object(s) not deleted after " + + MAX_DELETE_RETRIES + + " attempts in bucket " + + bucketName); } } @@ -1222,13 +1242,15 @@ public void testRemoveObjects(String testTags, List results throws Exception { String methodName = "removeObjects()"; long startTime = System.currentTimeMillis(); + boolean succeeded = false; try { removeObjects(bucketName, results); mintSuccessLog(methodName, testTags, startTime); + succeeded = true; } catch (Exception e) { handleException(methodName, testTags, startTime, e); } finally { - removeObjects(bucketName, results); + if (!succeeded) removeObjects(bucketName, results); } } From fb4e8d61e6817c768e35f861ebbabb0e7fa4c000 Mon Sep 17 00:00:00 2001 From: Allan Roger Reid Date: Mon, 27 Apr 2026 20:46:53 +0000 Subject: [PATCH 04/11] fix: chunk batch deletes to avoid SDK early-abort on transient errors MinioAsyncClient.removeObjects() sets completed=true as soon as any 1000-object chunk returns errors, silently dropping all subsequent chunks in that call. When toDelete exceeds 1000 entries and a transient error hits the first chunk, objects in later chunks are never attempted and not carried into the retry set, leaving the bucket non-empty. Fix by iterating toDelete in DELETE_BATCH_SIZE slices and issuing a separate removeObjects() call per slice so each slice is always fully processed regardless of errors in other slices. --- functional/TestMinioClient.java | 41 +++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/functional/TestMinioClient.java b/functional/TestMinioClient.java index 7e95903f4..59fe6d001 100644 --- a/functional/TestMinioClient.java +++ b/functional/TestMinioClient.java @@ -142,6 +142,9 @@ justification = "Allow catching super class Exception since it's tests") public class TestMinioClient extends TestArgs { private static final int MAX_DELETE_RETRIES = 5; + // Matches the SDK's internal chunk size: MinioAsyncClient sets completed=true on the + // first chunk that returns errors, dropping remaining chunks in the same call. + private static final int DELETE_BATCH_SIZE = 1000; private static final Set TRANSIENT_DELETE_CODES = Collections.unmodifiableSet( new HashSet<>( @@ -1058,25 +1061,29 @@ public void removeObjects(String bucketName, List results) } Set retryNames = new HashSet<>(); IOException nonTransientErr = null; - for (Result r : - client.removeObjects( - RemoveObjectsArgs.builder().bucket(bucketName).objects(toDelete).build())) { - DeleteResult.Error err = r.get(); - String code = err.code(); - if (!TRANSIENT_DELETE_CODES.contains(code)) { - if (nonTransientErr == null) { - nonTransientErr = - new IOException( - "non-transient delete error '" - + code - + "' on " - + err.objectName() - + " in bucket " - + bucketName); + for (int i = 0; i < toDelete.size(); i += DELETE_BATCH_SIZE) { + List chunk = + toDelete.subList(i, Math.min(i + DELETE_BATCH_SIZE, toDelete.size())); + for (Result r : + client.removeObjects( + RemoveObjectsArgs.builder().bucket(bucketName).objects(chunk).build())) { + DeleteResult.Error err = r.get(); + String code = err.code(); + if (!TRANSIENT_DELETE_CODES.contains(code)) { + if (nonTransientErr == null) { + nonTransientErr = + new IOException( + "non-transient delete error '" + + code + + "' on " + + err.objectName() + + " in bucket " + + bucketName); + } + continue; // drain remaining response before throwing } - continue; // drain remaining response before throwing + retryNames.add(err.objectName()); } - retryNames.add(err.objectName()); } if (nonTransientErr != null) throw nonTransientErr; // All versions re-queued because DeleteResult.Error lacks versionId; From 34820c7c56e998ec6d89b25ae612a2604a67e8bd Mon Sep 17 00:00:00 2001 From: Allan Roger Reid Date: Mon, 27 Apr 2026 20:59:41 +0000 Subject: [PATCH 05/11] fix: throw after draining chunk, not after all chunks After capturing a non-transient error, the previous code continued issuing removeObjects() calls for all remaining chunks before throwing. Move the nonTransientErr check to after each individual chunk's response is drained so no further delete RPCs are made once a fatal error is seen. --- functional/TestMinioClient.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functional/TestMinioClient.java b/functional/TestMinioClient.java index 59fe6d001..99f9d233c 100644 --- a/functional/TestMinioClient.java +++ b/functional/TestMinioClient.java @@ -1080,12 +1080,12 @@ public void removeObjects(String bucketName, List results) + " in bucket " + bucketName); } - continue; // drain remaining response before throwing + continue; // drain current chunk's response before throwing } retryNames.add(err.objectName()); } + if (nonTransientErr != null) throw nonTransientErr; } - if (nonTransientErr != null) throw nonTransientErr; // All versions re-queued because DeleteResult.Error lacks versionId; // already-deleted versions return NoSuchVersion, filtered upstream by MinioAsyncClient. toDelete = new ArrayList<>(); From 5efbf4080ff5875a3d9efaa0ff1b094b90f51bbd Mon Sep 17 00:00:00 2001 From: Allan Roger Reid Date: Mon, 27 Apr 2026 21:23:26 +0000 Subject: [PATCH 06/11] fix: address Copilot review feedback on removeObjects retry logic Rename MAX_DELETE_RETRIES to MAX_DELETE_ATTEMPTS to better reflect that the constant bounds attempts, not retries. Include err.message() in the non-transient IOException for richer diagnostics. Track failedNames across iterations and include a 5-object sample in the exhaustion IOException to aid debugging when all retries are consumed. --- functional/TestMinioClient.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/functional/TestMinioClient.java b/functional/TestMinioClient.java index 99f9d233c..c51e42e4b 100644 --- a/functional/TestMinioClient.java +++ b/functional/TestMinioClient.java @@ -141,7 +141,7 @@ value = "REC", justification = "Allow catching super class Exception since it's tests") public class TestMinioClient extends TestArgs { - private static final int MAX_DELETE_RETRIES = 5; + private static final int MAX_DELETE_ATTEMPTS = 5; // Matches the SDK's internal chunk size: MinioAsyncClient sets completed=true on the // first chunk that returns errors, dropping remaining chunks in the same call. private static final int DELETE_BATCH_SIZE = 1000; @@ -1050,7 +1050,8 @@ public void removeObjects(String bucketName, List results) results.stream() .map(r -> new DeleteRequest.Object(r.object(), r.versionId())) .collect(Collectors.toList()); - for (int attempt = 0; attempt < MAX_DELETE_RETRIES && !toDelete.isEmpty(); attempt++) { + Set failedNames = Collections.emptySet(); + for (int attempt = 0; attempt < MAX_DELETE_ATTEMPTS && !toDelete.isEmpty(); attempt++) { if (attempt > 0) { try { Thread.sleep(500L << attempt); // 1s / 2s / 4s / 8s @@ -1075,7 +1076,9 @@ public void removeObjects(String bucketName, List results) new IOException( "non-transient delete error '" + code - + "' on " + + "': " + + err.message() + + " on " + err.objectName() + " in bucket " + bucketName); @@ -1088,6 +1091,7 @@ public void removeObjects(String bucketName, List results) } // All versions re-queued because DeleteResult.Error lacks versionId; // already-deleted versions return NoSuchVersion, filtered upstream by MinioAsyncClient. + failedNames = retryNames; toDelete = new ArrayList<>(); for (String name : retryNames) { for (ObjectWriteResponse res : byName.getOrDefault(name, Collections.emptyList())) { @@ -1099,9 +1103,11 @@ public void removeObjects(String bucketName, List results) throw new IOException( toDelete.size() + " object(s) not deleted after " - + MAX_DELETE_RETRIES + + MAX_DELETE_ATTEMPTS + " attempts in bucket " - + bucketName); + + bucketName + + "; sample: " + + failedNames.stream().limit(5).collect(Collectors.toList())); } } From 91890504e1f30858642e1ab279abc2bba6dfac64 Mon Sep 17 00:00:00 2001 From: Allan Roger Reid Date: Mon, 27 Apr 2026 21:31:11 +0000 Subject: [PATCH 07/11] fix: suppress cleanup exception in testRemoveObjects finally block If removeObjects() in the try block throws, handleException() rethrows it. A second throw from the finally cleanup would then mask the original exception, making the test failure harder to diagnose. Wrap the cleanup call in try/catch so the original exception propagates unmasked. --- functional/TestMinioClient.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/functional/TestMinioClient.java b/functional/TestMinioClient.java index c51e42e4b..d4e9c3315 100644 --- a/functional/TestMinioClient.java +++ b/functional/TestMinioClient.java @@ -1263,7 +1263,13 @@ public void testRemoveObjects(String testTags, List results } catch (Exception e) { handleException(methodName, testTags, startTime, e); } finally { - if (!succeeded) removeObjects(bucketName, results); + if (!succeeded) { + try { + removeObjects(bucketName, results); + } catch (Exception ignored) { + // suppress so the original test exception propagates unmasked + } + } } } From d6ed5e8e2160c04db7790cefd6b3173f18dbee34 Mon Sep 17 00:00:00 2001 From: Allan Roger Reid Date: Mon, 27 Apr 2026 21:36:25 +0000 Subject: [PATCH 08/11] fix: suppress DE_MIGHT_IGNORE SpotBugs finding in testRemoveObjects The cleanup catch block in the finally clause intentionally discards the exception so the original test failure propagates unmasked. Suppress the DE_MIGHT_IGNORE finding at the method level. --- functional/TestMinioClient.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/functional/TestMinioClient.java b/functional/TestMinioClient.java index d4e9c3315..9b9bcb9e4 100644 --- a/functional/TestMinioClient.java +++ b/functional/TestMinioClient.java @@ -1251,6 +1251,10 @@ public void removeObject() throws Exception { RemoveObjectArgs.builder().bucket(bucketName).object(getRandomName()).build()); } + @edu.umd.cs.findbugs.annotations.SuppressFBWarnings( + value = "DE_MIGHT_IGNORE", + justification = + "Cleanup exception suppressed intentionally so original test failure propagates") public void testRemoveObjects(String testTags, List results) throws Exception { String methodName = "removeObjects()"; From 039724d3640ee84510c5d7ea8c622fde3cf0b5d3 Mon Sep 17 00:00:00 2001 From: Allan Roger Reid Date: Tue, 28 Apr 2026 12:33:47 +0000 Subject: [PATCH 09/11] fix: simplify removeObjects retry by retrying full set on transient failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace byName index, retryNames set, and toDelete rebuild loop with a single anyTransient boolean flag. Retry the full object list on transient errors rather than narrowing to failing names — already-deleted objects are silently ignored by S3 and MinioAsyncClient filters NoSuchVersion. --- functional/TestMinioClient.java | 37 ++++++++++----------------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/functional/TestMinioClient.java b/functional/TestMinioClient.java index 9b9bcb9e4..a24cc6004 100644 --- a/functional/TestMinioClient.java +++ b/functional/TestMinioClient.java @@ -1041,17 +1041,12 @@ public List createObjects(String bucketName, int count, int } public void removeObjects(String bucketName, List results) throws Exception { - // DeleteResult.Error has no versionId; keyed by name to rebuild versioned retry batches. - Map> byName = new HashMap<>(); - for (ObjectWriteResponse res : results) { - byName.computeIfAbsent(res.object(), k -> new ArrayList<>()).add(res); - } - List toDelete = + List objects = results.stream() .map(r -> new DeleteRequest.Object(r.object(), r.versionId())) .collect(Collectors.toList()); - Set failedNames = Collections.emptySet(); - for (int attempt = 0; attempt < MAX_DELETE_ATTEMPTS && !toDelete.isEmpty(); attempt++) { + boolean anyTransient = false; + for (int attempt = 0; attempt < MAX_DELETE_ATTEMPTS; attempt++) { if (attempt > 0) { try { Thread.sleep(500L << attempt); // 1s / 2s / 4s / 8s @@ -1060,11 +1055,11 @@ public void removeObjects(String bucketName, List results) throw ie; } } - Set retryNames = new HashSet<>(); + anyTransient = false; IOException nonTransientErr = null; - for (int i = 0; i < toDelete.size(); i += DELETE_BATCH_SIZE) { + for (int i = 0; i < objects.size(); i += DELETE_BATCH_SIZE) { List chunk = - toDelete.subList(i, Math.min(i + DELETE_BATCH_SIZE, toDelete.size())); + objects.subList(i, Math.min(i + DELETE_BATCH_SIZE, objects.size())); for (Result r : client.removeObjects( RemoveObjectsArgs.builder().bucket(bucketName).objects(chunk).build())) { @@ -1085,29 +1080,19 @@ public void removeObjects(String bucketName, List results) } continue; // drain current chunk's response before throwing } - retryNames.add(err.objectName()); + anyTransient = true; } if (nonTransientErr != null) throw nonTransientErr; } - // All versions re-queued because DeleteResult.Error lacks versionId; - // already-deleted versions return NoSuchVersion, filtered upstream by MinioAsyncClient. - failedNames = retryNames; - toDelete = new ArrayList<>(); - for (String name : retryNames) { - for (ObjectWriteResponse res : byName.getOrDefault(name, Collections.emptyList())) { - toDelete.add(new DeleteRequest.Object(res.object(), res.versionId())); - } - } + if (!anyTransient) break; } - if (!toDelete.isEmpty()) { + if (anyTransient) { throw new IOException( - toDelete.size() + results.size() + " object(s) not deleted after " + MAX_DELETE_ATTEMPTS + " attempts in bucket " - + bucketName - + "; sample: " - + failedNames.stream().limit(5).collect(Collectors.toList())); + + bucketName); } } From 246c1a67de7149aa1c8ebabcab0d88628b1951f8 Mon Sep 17 00:00:00 2001 From: Allan Roger Reid Date: Sat, 2 May 2026 18:35:23 -0700 Subject: [PATCH 10/11] fix: simplify removeObjects retry by retrying full set on transient failure --- functional/TestMinioClient.java | 45 ++++++++++++--------------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/functional/TestMinioClient.java b/functional/TestMinioClient.java index a24cc6004..2f215c259 100644 --- a/functional/TestMinioClient.java +++ b/functional/TestMinioClient.java @@ -142,9 +142,6 @@ justification = "Allow catching super class Exception since it's tests") public class TestMinioClient extends TestArgs { private static final int MAX_DELETE_ATTEMPTS = 5; - // Matches the SDK's internal chunk size: MinioAsyncClient sets completed=true on the - // first chunk that returns errors, dropping remaining chunks in the same call. - private static final int DELETE_BATCH_SIZE = 1000; private static final Set TRANSIENT_DELETE_CODES = Collections.unmodifiableSet( new HashSet<>( @@ -1056,33 +1053,23 @@ public void removeObjects(String bucketName, List results) } } anyTransient = false; - IOException nonTransientErr = null; - for (int i = 0; i < objects.size(); i += DELETE_BATCH_SIZE) { - List chunk = - objects.subList(i, Math.min(i + DELETE_BATCH_SIZE, objects.size())); - for (Result r : - client.removeObjects( - RemoveObjectsArgs.builder().bucket(bucketName).objects(chunk).build())) { - DeleteResult.Error err = r.get(); - String code = err.code(); - if (!TRANSIENT_DELETE_CODES.contains(code)) { - if (nonTransientErr == null) { - nonTransientErr = - new IOException( - "non-transient delete error '" - + code - + "': " - + err.message() - + " on " - + err.objectName() - + " in bucket " - + bucketName); - } - continue; // drain current chunk's response before throwing - } - anyTransient = true; + for (Result r : + client.removeObjects( + RemoveObjectsArgs.builder().bucket(bucketName).objects(objects).build())) { + DeleteResult.Error err = r.get(); + String code = err.code(); + if (!TRANSIENT_DELETE_CODES.contains(code)) { + throw new IOException( + "non-transient delete error '" + + code + + "': " + + err.message() + + " on " + + err.objectName() + + " in bucket " + + bucketName); } - if (nonTransientErr != null) throw nonTransientErr; + anyTransient = true; } if (!anyTransient) break; } From 9b7c558739005d93686453778e2f7c552a1e5200 Mon Sep 17 00:00:00 2001 From: Allan Roger Reid Date: Thu, 7 May 2026 15:55:37 +0000 Subject: [PATCH 11/11] fix: simplify testRemoveObjects and use Exception in removeObjects Address review feedback on PR #1700: - Use Exception instead of IOException in removeObjects() helper for consistency with the rest of the test class. - Remove the finally/succeeded retry in testRemoveObjects() now that removeObjects() retries transient failures internally; drops the associated DE_MIGHT_IGNORE SpotBugs suppression. --- functional/TestMinioClient.java | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/functional/TestMinioClient.java b/functional/TestMinioClient.java index 2f215c259..fcf9fc9f6 100644 --- a/functional/TestMinioClient.java +++ b/functional/TestMinioClient.java @@ -113,7 +113,6 @@ import io.minio.messages.Tags; import io.minio.messages.VersioningConfiguration; import java.io.ByteArrayInputStream; -import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -1059,7 +1058,7 @@ public void removeObjects(String bucketName, List results) DeleteResult.Error err = r.get(); String code = err.code(); if (!TRANSIENT_DELETE_CODES.contains(code)) { - throw new IOException( + throw new Exception( "non-transient delete error '" + code + "': " @@ -1074,7 +1073,7 @@ public void removeObjects(String bucketName, List results) if (!anyTransient) break; } if (anyTransient) { - throw new IOException( + throw new Exception( results.size() + " object(s) not deleted after " + MAX_DELETE_ATTEMPTS @@ -1223,29 +1222,15 @@ public void removeObject() throws Exception { RemoveObjectArgs.builder().bucket(bucketName).object(getRandomName()).build()); } - @edu.umd.cs.findbugs.annotations.SuppressFBWarnings( - value = "DE_MIGHT_IGNORE", - justification = - "Cleanup exception suppressed intentionally so original test failure propagates") public void testRemoveObjects(String testTags, List results) throws Exception { String methodName = "removeObjects()"; long startTime = System.currentTimeMillis(); - boolean succeeded = false; try { removeObjects(bucketName, results); mintSuccessLog(methodName, testTags, startTime); - succeeded = true; } catch (Exception e) { handleException(methodName, testTags, startTime, e); - } finally { - if (!succeeded) { - try { - removeObjects(bucketName, results); - } catch (Exception ignored) { - // suppress so the original test exception propagates unmasked - } - } } }