feat: update Storage.createFrom(BlobInfo, Path) to have 150% higher throughput

google-cloud-storage/src/main/java/com/google/cloud/storage/ByteRangeSpec.java

@@ -16,6 +16,9 @@
 
 package com.google.cloud.storage;
 
+import static com.google.api.client.util.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkArgument;
+
 import com.google.api.core.InternalApi;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.MoreObjects.ToStringHelper;
@@ -124,6 +127,20 @@ static ByteRangeSpec explicitClosed(
     return create(beginOffset, endOffsetInclusive, LeftClosedRightClosedByteRangeSpec::new);
   }
 
+  static ByteRangeSpec parse(String string) {
+    checkNotNull(string, "Range header is null");
+    checkArgument(string.startsWith("bytes="), "malformed Range header value: %s", string);
+
+    int i = string.indexOf('-');
+    String minS = string.substring(6, i);
+    String maxS = string.substring(i + 1);
+
+    long min = Long.parseLong(minS);
+    long max = Long.parseLong(maxS);
+
+    return explicitClosed(min, max);
+  }
+
   private static ByteRangeSpec create(
       @Nullable Long beginOffset,
       @Nullable Long length,

google-cloud-storage/src/main/java/com/google/cloud/storage/ByteSizeConstants.java

@@ -23,10 +23,16 @@ final class ByteSizeConstants {
   static final int _256KiB = 256 * _1KiB;
   static final int _384KiB = 384 * _1KiB;
   static final int _512KiB = 512 * _1KiB;
+  static final int _768KiB = 768 * _1KiB;
   static final int _1MiB = 1024 * _1KiB;
   static final int _2MiB = 2 * _1MiB;
   static final int _16MiB = 16 * _1MiB;
   static final int _32MiB = 32 * _1MiB;
 
+  static final long _128KiBL = 131072L;
+  static final long _256KiBL = 262144L;
+  static final long _512KiBL = 524288L;
+  static final long _768KiBL = 786432L;
+
   private ByteSizeConstants() {}
 }

google-cloud-storage/src/main/java/com/google/cloud/storage/HttpClientContext.java

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import com.google.api.client.http.HttpHeaders;
+import com.google.api.client.http.HttpRequestFactory;
+import com.google.api.client.json.JsonObjectParser;
+import com.google.api.client.util.ObjectParser;
+import com.google.cloud.storage.spi.v1.StorageRpc;
+import io.opencensus.trace.Span;
+import io.opencensus.trace.Tracer;
+import io.opencensus.trace.Tracing;
+import java.util.List;
+import org.checkerframework.checker.nullness.qual.NonNull;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
+final class HttpClientContext {
+
+  private final HttpRequestFactory requestFactory;
+  private final ObjectParser objectParser;
+  private final Tracer tracer;
+
+  private HttpClientContext(
+      HttpRequestFactory requestFactory, ObjectParser objectParser, Tracer tracer) {
+    this.requestFactory = requestFactory;
+    this.objectParser = objectParser;
+    this.tracer = tracer;
+  }
+
+  @SuppressWarnings({"unchecked", "SameParameterValue"})
+  static @Nullable String firstHeaderValue(
+      @NonNull HttpHeaders headers, @NonNull String headerName) {
+    Object v = headers.get(headerName);
+    // HttpHeaders doesn't type its get method, so we have to jump through hoops here
+    if (v instanceof List) {
+      List<String> list = (List<String>) v;
+      return list.get(0);
+    } else {
+      return null;
+    }
+  }
+
+  public HttpRequestFactory getRequestFactory() {
+    return requestFactory;
+  }
+
+  public ObjectParser getObjectParser() {
+    return objectParser;
+  }
+
+  public Tracer getTracer() {
+    return tracer;
+  }
+
+  public Span startSpan(String name) {
+    // record events is hardcoded to true in HttpStorageRpc, preserve it here
+    return tracer.spanBuilder(name).setRecordEvents(true).startSpan();
+  }
+
+  static HttpClientContext from(StorageRpc storageRpc) {
+    return new HttpClientContext(
+        storageRpc.getStorage().getRequestFactory(),
+        storageRpc.getStorage().getObjectParser(),
+        Tracing.getTracer());
+  }
+
+  public static HttpClientContext of(
+      HttpRequestFactory requestFactory, JsonObjectParser jsonObjectParser) {
+    return new HttpClientContext(requestFactory, jsonObjectParser, Tracing.getTracer());
+  }
+}

google-cloud-storage/src/main/java/com/google/cloud/storage/HttpContentRange.java

@@ -0,0 +1,249 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.storage;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.common.base.MoreObjects;
+import java.util.Objects;
+import java.util.function.UnaryOperator;
+
+abstract class HttpContentRange {
+
+  private final boolean finalizing;
+
+  private HttpContentRange(boolean finalizing) {
+    this.finalizing = finalizing;
+  }
+
+  public abstract String getHeaderValue();
+
+  public boolean isFinalizing() {
+    return finalizing;
+  }
+
+  static Total of(ByteRangeSpec spec, long size) {
+    checkArgument(size >= 0, "size must be >= 0");
+    checkArgument(size >= spec.endOffsetInclusive(), "size must be >= end");
+    return new Total(spec, size);
+  }
+
+  static Incomplete of(ByteRangeSpec spec) {
+    return new Incomplete(spec);
+  }
+
+  static Size of(long size) {
+    checkArgument(size >= 0, "size must be >= 0");
+    return new Size(size);
+  }
+
+  static Query query() {
+    return Query.INSTANCE;
+  }
+
+  static HttpContentRange parse(String string) {
+    if ("bytes */*".equals(string)) {
+      return HttpContentRange.query();
+    } else if (string.startsWith("bytes */")) {
+      return HttpContentRange.of(Long.parseLong(string.substring(8)));
+    } else {
+      int idxDash = string.indexOf('-');
+      int idxSlash = string.indexOf('/');
+
+      String beginS = string.substring(6, idxDash);
+      String endS = string.substring(idxDash + 1, idxSlash);
+      long begin = Long.parseLong(beginS);
+      long end = Long.parseLong(endS);
+      if (string.endsWith("/*")) {
+        return HttpContentRange.of(ByteRangeSpec.explicitClosed(begin, end));
+      } else {
+        String sizeS = string.substring(idxSlash + 1);
+        long size = Long.parseLong(sizeS);
+        return HttpContentRange.of(ByteRangeSpec.explicitClosed(begin, end), size);
+      }
+    }
+  }
+
+  static final class Incomplete extends HttpContentRange implements HasRange<Incomplete> {
+
+    private final ByteRangeSpec spec;
+
+    private Incomplete(ByteRangeSpec spec) {
+      super(false);
+      this.spec = spec;
+    }
+
+    @Override
+    public String getHeaderValue() {
+      return String.format("bytes %d-%d/*", spec.beginOffset(), spec.endOffsetInclusive());
+    }
+
+    @Override
+    public ByteRangeSpec range() {
+      return spec;
+    }
+
+    @Override
+    public Incomplete map(UnaryOperator<ByteRangeSpec> f) {
+      return new Incomplete(f.apply(spec));
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (!(o instanceof Incomplete)) {
+        return false;
+      }
+      Incomplete that = (Incomplete) o;
+      return Objects.equals(spec, that.spec);
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(spec);
+    }
+
+    @Override
+    public String toString() {
+      return MoreObjects.toStringHelper(this).add("spec", spec).toString();
+    }
+  }
+
+  static final class Total extends HttpContentRange implements HasRange<Total>, HasSize {
+
+    private final ByteRangeSpec spec;
+    private final long size;
+
+    private Total(ByteRangeSpec spec, long size) {
+      super(true);
+      this.spec = spec;
+      this.size = size;
+    }
+
+    @Override
+    public String getHeaderValue() {
+      return String.format("bytes %d-%d/%d", spec.beginOffset(), spec.endOffsetInclusive(), size);
+    }
+
+    @Override
+    public long getSize() {
+      return size;
+    }
+
+    @Override
+    public ByteRangeSpec range() {
+      return spec;
+    }
+
+    @Override
+    public Total map(UnaryOperator<ByteRangeSpec> f) {
+      return new Total(f.apply(spec), size);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (!(o instanceof Total)) {
+        return false;
+      }
+      Total total = (Total) o;
+      return size == total.size && Objects.equals(spec, total.spec);
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(spec, size);
+    }
+
+    @Override
+    public String toString() {
+      return MoreObjects.toStringHelper(this).add("spec", spec).add("size", size).toString();
+    }
+  }
+
+  static final class Size extends HttpContentRange implements HasSize {
+
+    private final long size;
+
+    private Size(long size) {
+      super(true);
+      this.size = size;
+    }
+
+    @Override
+    public String getHeaderValue() {
+      return String.format("bytes */%d", size);
+    }
+
+    @Override
+    public long getSize() {
+      return size;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (!(o instanceof Size)) {
+        return false;
+      }
+      Size size1 = (Size) o;
+      return size == size1.size;
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(size);
+    }
+
+    @Override
+    public String toString() {
+      return MoreObjects.toStringHelper(this).add("size", size).toString();
+    }
+  }
+
+  static final class Query extends HttpContentRange {
+
+    private static final Query INSTANCE = new Query();
+
+    private Query() {
+      super(false);
+    }
+
+    @Override
+    public String getHeaderValue() {
+      return "bytes */*";
+    }
+  }
+
+  interface HasRange<T extends HttpContentRange> {
+
+    ByteRangeSpec range();
+
+    T map(UnaryOperator<ByteRangeSpec> f);
+  }
+
+  interface HasSize {
+
+    long getSize();
+  }
+}