Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add new Storage#moveBlob method to atomically rename an object #2882

Merged
merged 2 commits into from
Jan 22, 2025
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions google-cloud-storage/clirr-ignored-differences.xml
Original file line number Diff line number Diff line change
Expand Up @@ -108,5 +108,16 @@
<method>io.opentelemetry.api.OpenTelemetry getOpenTelemetry()</method>
</difference>

<!-- Move Object -->
<difference>
<differenceType>7012</differenceType>
<className>com/google/cloud/storage/Storage</className>
<method>* moveBlob(*)</method>
</difference>
<difference>
<differenceType>7012</differenceType>
<className>com/google/cloud/storage/spi/v1/StorageRpc</className>
<method>* moveObject(*)</method>
</difference>

</differences>
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import com.google.storage.v2.ListBucketsRequest;
import com.google.storage.v2.ListObjectsRequest;
import com.google.storage.v2.LockBucketRetentionPolicyRequest;
import com.google.storage.v2.MoveObjectRequest;
import com.google.storage.v2.QueryWriteStatusRequest;
import com.google.storage.v2.ReadObjectRequest;
import com.google.storage.v2.RestoreObjectRequest;
Expand Down Expand Up @@ -124,6 +125,12 @@ public ResultRetryAlgorithm<?> getFor(RewriteObjectRequest req) {
: retryStrategy.getNonidempotentHandler();
}

public ResultRetryAlgorithm<?> getFor(MoveObjectRequest req) {
return req.hasIfGenerationMatch()
? retryStrategy.getIdempotentHandler()
: retryStrategy.getNonidempotentHandler();
}

public ResultRetryAlgorithm<?> getFor(SetIamPolicyRequest req) {
if (req.getPolicy().getEtag().equals(ByteString.empty())) {
return retryStrategy.getNonidempotentHandler();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@
import com.google.storage.v2.ListObjectsRequest;
import com.google.storage.v2.ListObjectsResponse;
import com.google.storage.v2.LockBucketRetentionPolicyRequest;
import com.google.storage.v2.MoveObjectRequest;
import com.google.storage.v2.Object;
import com.google.storage.v2.ObjectAccessControl;
import com.google.storage.v2.ReadObjectRequest;
Expand Down Expand Up @@ -1418,6 +1419,31 @@ public BlobWriteSession blobWriteSession(BlobInfo info, BlobWriteOption... optio
return BlobWriteSessions.of(writableByteChannelSession);
}

@Override
public Blob moveBlob(MoveBlobRequest request) {
Object srcObj = codecs.blobId().encode(request.getSource());
Object dstObj = codecs.blobId().encode(request.getTarget());
Opts<ObjectSourceOpt> srcOpts =
Opts.unwrap(request.getSourceOptions()).resolveFrom(request.getSource()).projectAsSource();
Opts<ObjectTargetOpt> dstOpts =
Opts.unwrap(request.getTargetOptions()).resolveFrom(request.getTarget());
MoveObjectRequest.Builder b =
MoveObjectRequest.newBuilder()
.setBucket(srcObj.getBucket())
.setSourceObject(srcObj.getName())
.setDestinationObject(dstObj.getName());

srcOpts.moveObjectsRequest().apply(b);
dstOpts.moveObjectsRequest().apply(b);

MoveObjectRequest req = b.build();
return Retrying.run(
getOptions(),
retryAlgorithmManager.getFor(req),
() -> storageClient.moveObjectCallable().call(req),
syntaxDecoders.blob);
}

@Override
public GrpcStorageOptions getOptions() {
return (GrpcStorageOptions) super.getOptions();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import com.google.cloud.storage.spi.v1.StorageRpc;
import com.google.cloud.storage.spi.v1.StorageRpc.RewriteRequest;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableMap;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -236,6 +237,14 @@ public ResultRetryAlgorithm<?> getForObjectsRewrite(RewriteRequest pb) {
: retryStrategy.getNonidempotentHandler();
}

public ResultRetryAlgorithm<?> getForObjectsMove(
ImmutableMap<StorageRpc.Option, ?> sourceOptions,
ImmutableMap<StorageRpc.Option, ?> targetOptions) {
return targetOptions.containsKey(StorageRpc.Option.IF_GENERATION_MATCH)
? retryStrategy.getIdempotentHandler()
: retryStrategy.getNonidempotentHandler();
}

public ResultRetryAlgorithm<?> getForObjectsCompose(
List<StorageObject> sources, StorageObject target, Map<StorageRpc.Option, ?> optionsMap) {
return optionsMap.containsKey(StorageRpc.Option.IF_GENERATION_MATCH)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1434,6 +1434,25 @@ public BlobWriteSession blobWriteSession(BlobInfo blobInfo, BlobWriteOption... o
}
}

@Override
public Blob moveBlob(MoveBlobRequest request) {
Span span =
tracer
.spanBuilder("moveBlob")
.setAttribute("gsutil.uri.source", request.getSource().toGsUtilUriWithGeneration())
.setAttribute("gsutil.uri.target", request.getTarget().toGsUtilUriWithGeneration())
.startSpan();
try (Scope ignore = span.makeCurrent()) {
return delegate.moveBlob(request);
} catch (Throwable t) {
span.recordException(t);
span.setStatus(StatusCode.ERROR, t.getClass().getSimpleName());
throw t;
} finally {
span.end();
}
}

@Override
public StorageOptions getOptions() {
return delegate.getOptions();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import com.google.cloud.storage.UnifiedOpts.ObjectSourceOpt;
import com.google.cloud.storage.UnifiedOpts.ObjectTargetOpt;
import com.google.cloud.storage.UnifiedOpts.Opts;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
Expand All @@ -73,6 +74,7 @@
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
Expand Down Expand Up @@ -2778,6 +2780,150 @@ public static Builder newBuilder() {
}
}

/**
* A class to contain all information needed for a Google Cloud Storage Object Move.
*
* @since 2.48.0
* @see Storage#moveBlob(MoveBlobRequest)
*/
@TransportCompatibility({Transport.HTTP, Transport.GRPC})
final class MoveBlobRequest {
private final BlobId source;
private final BlobId target;
private final ImmutableList<BlobSourceOption> sourceOptions;
private final ImmutableList<BlobTargetOption> targetOptions;

MoveBlobRequest(
BlobId source,
BlobId target,
ImmutableList<BlobSourceOption> sourceOptions,
ImmutableList<BlobTargetOption> targetOptions) {
this.source = source;
this.target = target;
this.sourceOptions = sourceOptions;
this.targetOptions = targetOptions;
}

public BlobId getSource() {
return source;
}

public BlobId getTarget() {
return target;
}

public List<BlobSourceOption> getSourceOptions() {
return sourceOptions;
}

public List<BlobTargetOption> getTargetOptions() {
return targetOptions;
}

public Builder toBuilder() {
return new Builder(source, target, sourceOptions, targetOptions);
}

public static Builder newBuilder() {
Copy link
Contributor

@JesseLovelace JesseLovelace Jan 21, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we maybe want to add something like MoveBlobRequest.of(BlobIdSource, BlobId target) for convenience?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could, but I sort of like having the builder methods that nudge people to at least think about preconditions. A move is only retryable if the target has a precondition set on it.

return new Builder();
}

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof MoveBlobRequest)) {
return false;
}
MoveBlobRequest that = (MoveBlobRequest) o;
return Objects.equals(source, that.source)
&& Objects.equals(target, that.target)
&& Objects.equals(sourceOptions, that.sourceOptions)
&& Objects.equals(targetOptions, that.targetOptions);
}

@Override
public int hashCode() {
return Objects.hash(source, target, sourceOptions, targetOptions);
}

@Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add("source", source)
.add("target", target)
.add("sourceOptions", sourceOptions)
.add("targetOptions", targetOptions)
.toString();
}

public static final class Builder {

private BlobId source;
private BlobId target;
private ImmutableList<BlobSourceOption> sourceOptions;
private ImmutableList<BlobTargetOption> targetOptions;

private Builder() {
this(null, null, ImmutableList.of(), ImmutableList.of());
}

private Builder(
BlobId source,
BlobId target,
ImmutableList<BlobSourceOption> sourceOptions,
ImmutableList<BlobTargetOption> targetOptions) {
this.source = source;
this.target = target;
this.sourceOptions = sourceOptions;
this.targetOptions = targetOptions;
}

public Builder setSource(BlobId source) {
this.source = requireNonNull(source, "source must be non null");
return this;
}

public Builder setTarget(BlobId target) {
this.target = requireNonNull(target, "target must be non null");
return this;
}

public Builder setSourceOptions(Iterable<BlobSourceOption> sourceOptions) {
this.sourceOptions =
ImmutableList.copyOf(requireNonNull(sourceOptions, "sourceOptions must be non null"));
return this;
}

public Builder setTargetOptions(Iterable<BlobTargetOption> targetOptions) {
this.targetOptions =
ImmutableList.copyOf(requireNonNull(targetOptions, "targetOptions must be non null"));
return this;
}

public Builder setSourceOptions(BlobSourceOption... sourceOptions) {
this.sourceOptions =
ImmutableList.copyOf(requireNonNull(sourceOptions, "sourceOptions must be non null"));
return this;
}

public Builder setTargetOptions(BlobTargetOption... targetOptions) {
this.targetOptions =
ImmutableList.copyOf(requireNonNull(targetOptions, "targetOptions must be non null"));
return this;
}

public MoveBlobRequest build() {
return new MoveBlobRequest(
requireNonNull(source, "source must be non null"),
requireNonNull(target, "target must be non null"),
sourceOptions,
targetOptions);
}
}
}

/**
* Creates a new bucket.
*
Expand Down Expand Up @@ -4882,4 +5028,15 @@ default void close() throws Exception {}
default BlobWriteSession blobWriteSession(BlobInfo blobInfo, BlobWriteOption... options) {
return throwGrpcOnly(fmtMethodName("blobWriteSession", BlobInfo.class, BlobWriteOption.class));
}

/**
* Atomically move an object from one name to another.
*
* <p>This new method is an atomic equivalent of the previous rewrite + delete, however without
* the ability to change metadata fields for the target object.
*
* @since 2.48.0
*/
@TransportCompatibility({Transport.HTTP, Transport.GRPC})
Blob moveBlob(MoveBlobRequest request);
}
Original file line number Diff line number Diff line change
Expand Up @@ -1696,6 +1696,27 @@ public BlobWriteSession blobWriteSession(BlobInfo blobInfo, BlobWriteOption... o
return BlobWriteSessions.of(writableByteChannelSession);
}

@Override
public Blob moveBlob(MoveBlobRequest request) {
Opts<ObjectSourceOpt> srcOpts =
Opts.unwrap(request.getSourceOptions()).resolveFrom(request.getSource()).projectAsSource();
Opts<ObjectTargetOpt> dstOpts =
Opts.unwrap(request.getTargetOptions()).resolveFrom(request.getTarget());
ImmutableMap<StorageRpc.Option, ?> sourceOptions = srcOpts.getRpcOptions();
ImmutableMap<StorageRpc.Option, ?> targetOptions = dstOpts.getRpcOptions();

return run(
retryAlgorithmManager.getForObjectsMove(sourceOptions, targetOptions),
() ->
storageRpc.moveObject(
request.getSource().getBucket(),
request.getSource().getName(),
request.getTarget().getName(),
sourceOptions,
targetOptions),
o -> codecs.blobInfo().decode(o).asBlob(this));
}

@Override
public BlobInfo internalCreateFrom(Path path, BlobInfo info, Opts<ObjectTargetOpt> opts)
throws IOException {
Expand Down
Loading
Loading