Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor iterator in removeObjects() API. #1076

Merged
merged 1 commit into from
Oct 19, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 24 additions & 48 deletions api/src/main/java/io/minio/MinioClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -1721,13 +1721,16 @@ public Iterable<Result<DeleteError>> removeObjects(RemoveObjectsArgs args) {
@Override
public Iterator<Result<DeleteError>> iterator() {
return new Iterator<Result<DeleteError>>() {
private Result<DeleteError> error;
private Iterator<DeleteError> errorIterator;
private Result<DeleteError> error = null;
private Iterator<DeleteError> errorIterator = null;
private boolean completed = false;
private Iterator<DeleteObject> objectIter = args.objects().iterator();

private synchronized void populate() {
List<DeleteError> errorList = null;
if (completed) {
return;
}

try {
List<DeleteObject> objectList = new LinkedList<>();
int i = 0;
Expand All @@ -1736,17 +1739,21 @@ private synchronized void populate() {
i++;
}

if (objectList.size() > 0) {
completed = objectList.size() == 0;
if (!completed) {
DeleteObjectsResponse response =
deleteObjects(
args.bucket(),
args.region(),
objectList,
args.quiet(),
true,
args.bypassGovernanceMode(),
args.extraHeaders(),
args.extraQueryParams());
errorList = response.result().errorList();
if (response.result().errorList().size() > 0) {
errorIterator = response.result().errorList().iterator();
completed = true;
}
}
} catch (ErrorResponseException
| InsufficientDataException
Expand All @@ -1757,66 +1764,35 @@ private synchronized void populate() {
| NoSuchAlgorithmException
| ServerException
| XmlParserException e) {
this.error = new Result<>(e);
} finally {
if (errorList != null) {
this.errorIterator = errorList.iterator();
} else {
this.errorIterator = new LinkedList<DeleteError>().iterator();
}
error = new Result<>(e);
completed = true;
}
}

@Override
public boolean hasNext() {
if (this.completed) {
return false;
}

if (this.error == null && this.errorIterator == null) {
while (error == null && errorIterator == null && !completed) {
populate();
}

if (this.error == null && this.errorIterator != null && !this.errorIterator.hasNext()) {
populate();
}

if (this.error != null) {
return true;
}

if (this.errorIterator.hasNext()) {
if (error != null || (errorIterator != null && errorIterator.hasNext())) {
return true;
}

this.completed = true;
return false;
return !completed;
}

@Override
public Result<DeleteError> next() {
if (this.completed) {
throw new NoSuchElementException();
}

if (this.error == null && this.errorIterator == null) {
populate();
}

if (this.error == null && this.errorIterator != null && !this.errorIterator.hasNext()) {
populate();
}

if (this.error != null) {
this.completed = true;
return this.error;
}
if (!hasNext()) throw new NoSuchElementException();

if (this.errorIterator.hasNext()) {
return new Result<>(this.errorIterator.next());
if (error != null) {
return error;
} else if (errorIterator != null && errorIterator.hasNext()) {
return new Result<>(errorIterator.next());
}

this.completed = true;
// This never happens.
throw new NoSuchElementException();
}

Expand Down
10 changes: 0 additions & 10 deletions api/src/main/java/io/minio/RemoveObjectsArgs.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
public class RemoveObjectsArgs extends BucketArgs {
private boolean bypassGovernanceMode;
private Iterable<DeleteObject> objects = new LinkedList<>();
private boolean quiet;

public boolean bypassGovernanceMode() {
return bypassGovernanceMode;
Expand All @@ -33,10 +32,6 @@ public Iterable<DeleteObject> objects() {
return objects;
}

public boolean quiet() {
return quiet;
}

public static Builder builder() {
return new Builder();
}
Expand All @@ -53,10 +48,5 @@ public Builder objects(Iterable<DeleteObject> objects) {
operations.add(args -> args.objects = objects);
return this;
}

public Builder quiet(boolean flag) {
operations.add(args -> args.quiet = flag);
return this;
}
}
}
52 changes: 39 additions & 13 deletions functional/FunctionalTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -1519,28 +1519,54 @@ public static void removeObject() throws Exception {
RemoveObjectArgs.builder().bucket(bucketName).object(getRandomName()).build());
}

public static void testRemoveObjects(String testTags, List<ObjectWriteResponse> results)
throws Exception {
String methodName = "removeObjects()";
long startTime = System.currentTimeMillis();
try {
removeObjects(bucketName, results);
mintSuccessLog(methodName, testTags, startTime);
} catch (Exception e) {
handleException(methodName, testTags, startTime, e);
} finally {
removeObjects(bucketName, results);
}
}

public static void removeObjects() throws Exception {
String methodName = "removeObjects()";
if (!mintEnv) {
System.out.println(methodName);
}

testRemoveObjects("[basic]", createObjects(bucketName, 3, 0));

String testTags = "[3005 objects]";
long startTime = System.currentTimeMillis();
String objectName = getRandomName();
List<ObjectWriteResponse> results = new LinkedList<>();
for (int i = 0; i < 3004; i++) {
results.add(
new ObjectWriteResponse(null, bucketName, null, objectName + "-" + i, null, null));
}
List<ObjectWriteResponse> existingObject = createObjects(bucketName, 1, 0);
results.addAll(existingObject);
testRemoveObjects(testTags, results);
try {
List<ObjectWriteResponse> results = null;
try {
results = createObjects(bucketName, 3, 0);
results.add(
new ObjectWriteResponse(null, bucketName, null, "nonexistent-object", null, null));
removeObjects(bucketName, results);
mintSuccessLog(methodName, null, startTime);
} finally {
if (results != null) {
removeObjects(bucketName, results);
}
client.statObject(
StatObjectArgs.builder()
.bucket(bucketName)
.object(existingObject.get(0).object())
.build());
handleException(
methodName,
testTags,
startTime,
new Exception("object " + existingObject.get(0).object() + " still exist"));
} catch (ErrorResponseException e) {
if (!e.errorResponse().code().equals("NoSuchKey")) {
throw e;
}
} catch (Exception e) {
handleException(methodName, null, startTime, e);
}
}

Expand Down