feat: [datalineage] add open lineage support (#10041)

* feat: add open lineage support fix: change `start_time` in message `.google.cloud.datacatalog.lineage.v1.LineageEvent` to `required` as intended by api PiperOrigin-RevId: 579762272 Source-Link: googleapis/googleapis@58878bd Source-Link: https://github.com/googleapis/googleapis-gen/commit/96a4d735b92f202fbdd2c62da2e21e63216bfa88 Copy-Tag: eyJwIjoiamF2YS1kYXRhbGluZWFnZS8uT3dsQm90LnlhbWwiLCJoIjoiOTZhNGQ3MzViOTJmMjAyZmJkZDJjNjJkYTJlMjFlNjMyMTZiZmE4OCJ9 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
googleapis · Nov 6, 2023 · ceb26a1 · ceb26a1
1 parent cc0e248
commit ceb26a1
Show file tree

Hide file tree

Showing 44 changed files with 4,060 additions and 475 deletions.
diff --git a/java-datalineage/README.md b/java-datalineage/README.md
@@ -23,7 +23,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file:
     <dependency>
       <groupId>com.google.cloud</groupId>
       <artifactId>libraries-bom</artifactId>
-      <version>26.23.0</version>
+      <version>26.26.0</version>
       <type>pom</type>
       <scope>import</scope>
     </dependency>
@@ -201,7 +201,7 @@ Java is a registered trademark of Oracle and/or its affiliates.
 [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-datalineage/java11.html
 [stability-image]: https://img.shields.io/badge/stability-preview-yellow
 [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-datalineage.svg
-[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-datalineage/0.16.0
+[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-datalineage/0.21.0
 [authentication]: https://github.com/googleapis/google-cloud-java#authentication
 [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes
 [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles

diff --git a/...loud-datalineage/src/main/java/com/google/cloud/datacatalog/lineage/v1/LineageClient.java b/...loud-datalineage/src/main/java/com/google/cloud/datacatalog/lineage/v1/LineageClient.java
@@ -34,6 +34,7 @@
 import com.google.longrunning.Operation;
 import com.google.protobuf.Empty;
 import com.google.protobuf.FieldMask;
+import com.google.protobuf.Struct;
 import java.io.IOException;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -56,9 +57,10 @@
  * // - It may require specifying regional endpoints when creating the service client as shown in
  * // https://cloud.google.com/java/docs/setup#configure_endpoints_for_the_client_library
  * try (LineageClient lineageClient = LineageClient.create()) {
- *   LocationName parent = LocationName.of("[PROJECT]", "[LOCATION]");
- *   Process process = Process.newBuilder().build();
- *   Process response = lineageClient.createProcess(parent, process);
+ *   String parent = "parent-995424086";
+ *   Struct openLineage = Struct.newBuilder().build();
+ *   ProcessOpenLineageRunEventResponse response =
+ *       lineageClient.processOpenLineageRunEvent(parent, openLineage);
  * }
  * }</pre>
  *
@@ -203,6 +205,111 @@ public final OperationsClient getHttpJsonOperationsClient() {
     return httpJsonOperationsClient;
   }
 
+  // AUTO-GENERATED DOCUMENTATION AND METHOD.
+  /**
+   * Creates new lineage events together with their parents: process and run. Updates the process
+   * and run if they already exist. Mapped from Open Lineage specification:
+   * https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json.
+   *
+   * <p>Sample code:
+   *
+   * <pre>{@code
+   * // This snippet has been automatically generated and should be regarded as a code template only.
+   * // It will require modifications to work:
+   * // - It may require correct/in-range values for request initialization.
+   * // - It may require specifying regional endpoints when creating the service client as shown in
+   * // https://cloud.google.com/java/docs/setup#configure_endpoints_for_the_client_library
+   * try (LineageClient lineageClient = LineageClient.create()) {
+   *   String parent = "parent-995424086";
+   *   Struct openLineage = Struct.newBuilder().build();
+   *   ProcessOpenLineageRunEventResponse response =
+   *       lineageClient.processOpenLineageRunEvent(parent, openLineage);
+   * }
+   * }</pre>
+   *
+   * @param parent Required. The name of the project and its location that should own the process,
+   *     run, and lineage event.
+   * @param openLineage Required. OpenLineage message following OpenLineage format:
+   *     https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json
+   * @throws com.google.api.gax.rpc.ApiException if the remote call fails
+   */
+  public final ProcessOpenLineageRunEventResponse processOpenLineageRunEvent(
+      String parent, Struct openLineage) {
+    ProcessOpenLineageRunEventRequest request =
+        ProcessOpenLineageRunEventRequest.newBuilder()
+            .setParent(parent)
+            .setOpenLineage(openLineage)
+            .build();
+    return processOpenLineageRunEvent(request);
+  }
+
+  // AUTO-GENERATED DOCUMENTATION AND METHOD.
+  /**
+   * Creates new lineage events together with their parents: process and run. Updates the process
+   * and run if they already exist. Mapped from Open Lineage specification:
+   * https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json.
+   *
+   * <p>Sample code:
+   *
+   * <pre>{@code
+   * // This snippet has been automatically generated and should be regarded as a code template only.
+   * // It will require modifications to work:
+   * // - It may require correct/in-range values for request initialization.
+   * // - It may require specifying regional endpoints when creating the service client as shown in
+   * // https://cloud.google.com/java/docs/setup#configure_endpoints_for_the_client_library
+   * try (LineageClient lineageClient = LineageClient.create()) {
+   *   ProcessOpenLineageRunEventRequest request =
+   *       ProcessOpenLineageRunEventRequest.newBuilder()
+   *           .setParent("parent-995424086")
+   *           .setOpenLineage(Struct.newBuilder().build())
+   *           .setRequestId("requestId693933066")
+   *           .build();
+   *   ProcessOpenLineageRunEventResponse response =
+   *       lineageClient.processOpenLineageRunEvent(request);
+   * }
+   * }</pre>
+   *
+   * @param request The request object containing all of the parameters for the API call.
+   * @throws com.google.api.gax.rpc.ApiException if the remote call fails
+   */
+  public final ProcessOpenLineageRunEventResponse processOpenLineageRunEvent(
+      ProcessOpenLineageRunEventRequest request) {
+    return processOpenLineageRunEventCallable().call(request);
+  }
+
+  // AUTO-GENERATED DOCUMENTATION AND METHOD.
+  /**
+   * Creates new lineage events together with their parents: process and run. Updates the process
+   * and run if they already exist. Mapped from Open Lineage specification:
+   * https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json.
+   *
+   * <p>Sample code:
+   *
+   * <pre>{@code
+   * // This snippet has been automatically generated and should be regarded as a code template only.
+   * // It will require modifications to work:
+   * // - It may require correct/in-range values for request initialization.
+   * // - It may require specifying regional endpoints when creating the service client as shown in
+   * // https://cloud.google.com/java/docs/setup#configure_endpoints_for_the_client_library
+   * try (LineageClient lineageClient = LineageClient.create()) {
+   *   ProcessOpenLineageRunEventRequest request =
+   *       ProcessOpenLineageRunEventRequest.newBuilder()
+   *           .setParent("parent-995424086")
+   *           .setOpenLineage(Struct.newBuilder().build())
+   *           .setRequestId("requestId693933066")
+   *           .build();
+   *   ApiFuture<ProcessOpenLineageRunEventResponse> future =
+   *       lineageClient.processOpenLineageRunEventCallable().futureCall(request);
+   *   // Do something.
+   *   ProcessOpenLineageRunEventResponse response = future.get();
+   * }
+   * }</pre>
+   */
+  public final UnaryCallable<ProcessOpenLineageRunEventRequest, ProcessOpenLineageRunEventResponse>
+      processOpenLineageRunEventCallable() {
+    return stub.processOpenLineageRunEventCallable();
+  }
+
   // AUTO-GENERATED DOCUMENTATION AND METHOD.
   /**
    * Creates a new process.
@@ -993,6 +1100,7 @@ public final Run updateRun(Run run, FieldMask updateMask) {
    *       UpdateRunRequest.newBuilder()
    *           .setRun(Run.newBuilder().build())
    *           .setUpdateMask(FieldMask.newBuilder().build())
+   *           .setAllowMissing(true)
    *           .build();
    *   Run response = lineageClient.updateRun(request);
    * }
@@ -1022,6 +1130,7 @@ public final Run updateRun(UpdateRunRequest request) {
    *       UpdateRunRequest.newBuilder()
    *           .setRun(Run.newBuilder().build())
    *           .setUpdateMask(FieldMask.newBuilder().build())
+   *           .setAllowMissing(true)
    *           .build();
    *   ApiFuture<Run> future = lineageClient.updateRunCallable().futureCall(request);
    *   // Do something.

diff --git a/...ud-datalineage/src/main/java/com/google/cloud/datacatalog/lineage/v1/LineageSettings.java b/...ud-datalineage/src/main/java/com/google/cloud/datacatalog/lineage/v1/LineageSettings.java
@@ -57,7 +57,7 @@
  * <p>The builder of this class is recursive, so contained classes are themselves builders. When
  * build() is called, the tree of builders is called to create the complete settings object.
  *
- * <p>For example, to set the total timeout of createProcess to 30 seconds:
+ * <p>For example, to set the total timeout of processOpenLineageRunEvent to 30 seconds:
  *
  * <pre>{@code
  * // This snippet has been automatically generated and should be regarded as a code template only.
@@ -67,10 +67,10 @@
  * // https://cloud.google.com/java/docs/setup#configure_endpoints_for_the_client_library
  * LineageSettings.Builder lineageSettingsBuilder = LineageSettings.newBuilder();
  * lineageSettingsBuilder
- *     .createProcessSettings()
+ *     .processOpenLineageRunEventSettings()
  *     .setRetrySettings(
  *         lineageSettingsBuilder
- *             .createProcessSettings()
+ *             .processOpenLineageRunEventSettings()
  *             .getRetrySettings()
  *             .toBuilder()
  *             .setTotalTimeout(Duration.ofSeconds(30))
@@ -81,6 +81,12 @@
 @Generated("by gapic-generator-java")
 public class LineageSettings extends ClientSettings<LineageSettings> {
 
+  /** Returns the object with the settings used for calls to processOpenLineageRunEvent. */
+  public UnaryCallSettings<ProcessOpenLineageRunEventRequest, ProcessOpenLineageRunEventResponse>
+      processOpenLineageRunEventSettings() {
+    return ((LineageStubSettings) getStubSettings()).processOpenLineageRunEventSettings();
+  }
+
   /** Returns the object with the settings used for calls to createProcess. */
   public UnaryCallSettings<CreateProcessRequest, Process> createProcessSettings() {
     return ((LineageStubSettings) getStubSettings()).createProcessSettings();
@@ -296,6 +302,13 @@ public Builder applyToAllUnaryMethods(
       return this;
     }
 
+    /** Returns the builder for the settings used for calls to processOpenLineageRunEvent. */
+    public UnaryCallSettings.Builder<
+            ProcessOpenLineageRunEventRequest, ProcessOpenLineageRunEventResponse>
+        processOpenLineageRunEventSettings() {
+      return getStubSettingsBuilder().processOpenLineageRunEventSettings();
+    }
+
     /** Returns the builder for the settings used for calls to createProcess. */
     public UnaryCallSettings.Builder<CreateProcessRequest, Process> createProcessSettings() {
       return getStubSettingsBuilder().createProcessSettings();

diff --git a/...oud-datalineage/src/main/java/com/google/cloud/datacatalog/lineage/v1/gapic_metadata.json b/...oud-datalineage/src/main/java/com/google/cloud/datacatalog/lineage/v1/gapic_metadata.json
@@ -49,6 +49,9 @@
             "ListRuns": {
               "methods": ["listRuns", "listRuns", "listRuns", "listRunsPagedCallable", "listRunsCallable"]
             },
+            "ProcessOpenLineageRunEvent": {
+              "methods": ["processOpenLineageRunEvent", "processOpenLineageRunEvent", "processOpenLineageRunEventCallable"]
+            },
             "SearchLinks": {
               "methods": ["searchLinks", "searchLinksPagedCallable", "searchLinksCallable"]
             },

diff --git a/...cloud-datalineage/src/main/java/com/google/cloud/datacatalog/lineage/v1/package-info.java b/...cloud-datalineage/src/main/java/com/google/cloud/datacatalog/lineage/v1/package-info.java
@@ -35,9 +35,10 @@
  * // - It may require specifying regional endpoints when creating the service client as shown in
  * // https://cloud.google.com/java/docs/setup#configure_endpoints_for_the_client_library
  * try (LineageClient lineageClient = LineageClient.create()) {
- *   LocationName parent = LocationName.of("[PROJECT]", "[LOCATION]");
- *   Process process = Process.newBuilder().build();
- *   Process response = lineageClient.createProcess(parent, process);
+ *   String parent = "parent-995424086";
+ *   Struct openLineage = Struct.newBuilder().build();
+ *   ProcessOpenLineageRunEventResponse response =
+ *       lineageClient.processOpenLineageRunEvent(parent, openLineage);
  * }
  * }</pre>
  */

diff --git a/...talineage/src/main/java/com/google/cloud/datacatalog/lineage/v1/stub/GrpcLineageStub.java b/...talineage/src/main/java/com/google/cloud/datacatalog/lineage/v1/stub/GrpcLineageStub.java
@@ -50,6 +50,8 @@
 import com.google.cloud.datacatalog.lineage.v1.ListRunsResponse;
 import com.google.cloud.datacatalog.lineage.v1.OperationMetadata;
 import com.google.cloud.datacatalog.lineage.v1.Process;
+import com.google.cloud.datacatalog.lineage.v1.ProcessOpenLineageRunEventRequest;
+import com.google.cloud.datacatalog.lineage.v1.ProcessOpenLineageRunEventResponse;
 import com.google.cloud.datacatalog.lineage.v1.Run;
 import com.google.cloud.datacatalog.lineage.v1.SearchLinksRequest;
 import com.google.cloud.datacatalog.lineage.v1.SearchLinksResponse;
@@ -72,6 +74,20 @@
  */
 @Generated("by gapic-generator-java")
 public class GrpcLineageStub extends LineageStub {
+  private static final MethodDescriptor<
+          ProcessOpenLineageRunEventRequest, ProcessOpenLineageRunEventResponse>
+      processOpenLineageRunEventMethodDescriptor =
+          MethodDescriptor
+              .<ProcessOpenLineageRunEventRequest, ProcessOpenLineageRunEventResponse>newBuilder()
+              .setType(MethodDescriptor.MethodType.UNARY)
+              .setFullMethodName(
+                  "google.cloud.datacatalog.lineage.v1.Lineage/ProcessOpenLineageRunEvent")
+              .setRequestMarshaller(
+                  ProtoUtils.marshaller(ProcessOpenLineageRunEventRequest.getDefaultInstance()))
+              .setResponseMarshaller(
+                  ProtoUtils.marshaller(ProcessOpenLineageRunEventResponse.getDefaultInstance()))
+              .build();
+
   private static final MethodDescriptor<CreateProcessRequest, Process>
       createProcessMethodDescriptor =
           MethodDescriptor.<CreateProcessRequest, Process>newBuilder()
@@ -227,6 +243,8 @@ public class GrpcLineageStub extends LineageStub {
                   ProtoUtils.marshaller(BatchSearchLinkProcessesResponse.getDefaultInstance()))
               .build();
 
+  private final UnaryCallable<ProcessOpenLineageRunEventRequest, ProcessOpenLineageRunEventResponse>
+      processOpenLineageRunEventCallable;
   private final UnaryCallable<CreateProcessRequest, Process> createProcessCallable;
   private final UnaryCallable<UpdateProcessRequest, Process> updateProcessCallable;
   private final UnaryCallable<GetProcessRequest, Process> getProcessCallable;
@@ -299,6 +317,18 @@ protected GrpcLineageStub(
     this.callableFactory = callableFactory;
     this.operationsStub = GrpcOperationsStub.create(clientContext, callableFactory);
 
+    GrpcCallSettings<ProcessOpenLineageRunEventRequest, ProcessOpenLineageRunEventResponse>
+        processOpenLineageRunEventTransportSettings =
+            GrpcCallSettings
+                .<ProcessOpenLineageRunEventRequest, ProcessOpenLineageRunEventResponse>newBuilder()
+                .setMethodDescriptor(processOpenLineageRunEventMethodDescriptor)
+                .setParamsExtractor(
+                    request -> {
+                      RequestParamsBuilder builder = RequestParamsBuilder.create();
+                      builder.add("parent", String.valueOf(request.getParent()));
+                      return builder.build();
+                    })
+                .build();
     GrpcCallSettings<CreateProcessRequest, Process> createProcessTransportSettings =
         GrpcCallSettings.<CreateProcessRequest, Process>newBuilder()
             .setMethodDescriptor(createProcessMethodDescriptor)
@@ -463,6 +493,11 @@ protected GrpcLineageStub(
                     })
                 .build();
 
+    this.processOpenLineageRunEventCallable =
+        callableFactory.createUnaryCallable(
+            processOpenLineageRunEventTransportSettings,
+            settings.processOpenLineageRunEventSettings(),
+            clientContext);
     this.createProcessCallable =
         callableFactory.createUnaryCallable(
             createProcessTransportSettings, settings.createProcessSettings(), clientContext);
@@ -559,6 +594,12 @@ public GrpcOperationsStub getOperationsStub() {
     return operationsStub;
   }
 
+  @Override
+  public UnaryCallable<ProcessOpenLineageRunEventRequest, ProcessOpenLineageRunEventResponse>
+      processOpenLineageRunEventCallable() {
+    return processOpenLineageRunEventCallable;
+  }
+
   @Override
   public UnaryCallable<CreateProcessRequest, Process> createProcessCallable() {
     return createProcessCallable;