From c039a83dc5644a3798eff1ce238ec8fd73b0286b Mon Sep 17 00:00:00 2001
From: Mattie Fu <mattiefu@google.com>
Date: Thu, 25 Aug 2022 15:31:58 -0400
Subject: [PATCH] test: add integration test for builtin metrics (#1360)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* test: add integration test for builtin metrics

* add license

* test on staging

* udpate

* address comments

* remove debugging

* fix dependency test

* update comment

* update integration test to only close client if it's not null

* explain why we're including grpc-xds

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
---
 google-cloud-bigtable-bom/pom.xml             |   1 +
 google-cloud-bigtable-deps-bom/pom.xml        |   1 +
 google-cloud-bigtable-stats/pom.xml           |   2 +-
 google-cloud-bigtable/pom.xml                 |  30 +++-
 .../bigtable/data/v2/it/BuiltinMetricsIT.java | 137 ++++++++++++++++++
 pom.xml                                       |  18 ++-
 6 files changed, 185 insertions(+), 4 deletions(-)
 create mode 100644 google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java

diff --git a/google-cloud-bigtable-bom/pom.xml b/google-cloud-bigtable-bom/pom.xml
index 805d5c34fd..51b0006b95 100644
--- a/google-cloud-bigtable-bom/pom.xml
+++ b/google-cloud-bigtable-bom/pom.xml
@@ -9,6 +9,7 @@
         <groupId>com.google.cloud</groupId>
         <artifactId>google-cloud-shared-config</artifactId>
         <version>1.5.1</version>
+        <relativePath/>
     </parent>
 
     <name>Google Cloud Bigtable BOM</name>
diff --git a/google-cloud-bigtable-deps-bom/pom.xml b/google-cloud-bigtable-deps-bom/pom.xml
index 19d705e47d..ca8ee09f9b 100644
--- a/google-cloud-bigtable-deps-bom/pom.xml
+++ b/google-cloud-bigtable-deps-bom/pom.xml
@@ -8,6 +8,7 @@
     <groupId>com.google.cloud</groupId>
     <artifactId>google-cloud-shared-config</artifactId>
     <version>1.5.1</version>
+    <relativePath/>
   </parent>
 
   <groupId>com.google.cloud</groupId>
diff --git a/google-cloud-bigtable-stats/pom.xml b/google-cloud-bigtable-stats/pom.xml
index 13a4a8be4c..c890860f28 100644
--- a/google-cloud-bigtable-stats/pom.xml
+++ b/google-cloud-bigtable-stats/pom.xml
@@ -59,7 +59,7 @@
         <dependency>
             <groupId>com.google.cloud</groupId>
             <artifactId>google-cloud-monitoring</artifactId>
-            <!-- Excluding http dependencies since they're not used and are causing conflict in google-cloud-bigtable -->
+            <!-- Excluding unused transitive dependencies to avoid conflict in google-cloud-bigtable -->
             <exclusions>
                 <exclusion>
                     <groupId>com.google.http-client</groupId>
diff --git a/google-cloud-bigtable/pom.xml b/google-cloud-bigtable/pom.xml
index 42c59583bc..1aebeea4d0 100644
--- a/google-cloud-bigtable/pom.xml
+++ b/google-cloud-bigtable/pom.xml
@@ -54,17 +54,25 @@
         <type>pom</type>
         <scope>import</scope>
       </dependency>
+      <dependency>
+        <groupId>com.google.cloud</groupId>
+        <artifactId>google-cloud-monitoring-bom</artifactId>
+        <version>3.4.1</version> <!-- needs to be in sync with google-cloud-bigtable-stats -->
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 
   <dependencies>
     <!-- NOTE: Dependencies are organized into two groups, production and test.
          Within a group, dependencies are sorted by (groupId, artifactId) -->
-
     <dependency>
       <groupId>com.google.cloud</groupId>
       <artifactId>google-cloud-bigtable-stats</artifactId>
-      <!-- Exclude all the shaded transitive dependencies -->
+      <!-- Exclude all dependencies that have been shaded. This is to workaround maven's immutable project structure:
+           after shading, the maven-shade-plugin tries to remove shaded dependencies, but it can't since the project
+           structure is immutable. So we have to manually exclude the shaded transitive dependencies manually. -->
       <exclusions>
         <exclusion>
           <groupId>io.opencensus</groupId>
@@ -254,6 +262,24 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <!-- cloud-monitoring client depends on grpc-xds, which has transitive dependency on opencensus-proto. We need to
+    use cloud-monitoring as a test dependency, which also marks the opencensus-proto as a test dependency. Promote
+    grpc-xds to a runtime dependency so that opencensus-proto will still be a runtime dependency in the flattened pom. -->
+    <dependency>
+      <groupId>io.grpc</groupId>
+      <artifactId>grpc-xds</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-monitoring</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.api.grpc</groupId>
+      <artifactId>proto-google-cloud-monitoring-v3</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>com.google.truth</groupId>
       <artifactId>truth</artifactId>
diff --git a/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java
new file mode 100644
index 0000000000..25ec8e442e
--- /dev/null
+++ b/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/it/BuiltinMetricsIT.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright 2022 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.cloud.bigtable.data.v2.it;
+
+import static com.google.common.truth.Truth.assertThat;
+import static com.google.common.truth.TruthJUnit.assume;
+
+import com.google.api.client.util.Lists;
+import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
+import com.google.cloud.bigtable.data.v2.models.Query;
+import com.google.cloud.bigtable.data.v2.models.Row;
+import com.google.cloud.bigtable.data.v2.models.RowMutation;
+import com.google.cloud.bigtable.test_helpers.env.EmulatorEnv;
+import com.google.cloud.bigtable.test_helpers.env.TestEnvRule;
+import com.google.cloud.monitoring.v3.MetricServiceClient;
+import com.google.monitoring.v3.ListTimeSeriesRequest;
+import com.google.monitoring.v3.ListTimeSeriesResponse;
+import com.google.monitoring.v3.ProjectName;
+import com.google.monitoring.v3.TimeInterval;
+import com.google.protobuf.util.Timestamps;
+import java.io.IOException;
+import java.time.Duration;
+import java.util.ArrayList;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+public class BuiltinMetricsIT {
+  @ClassRule public static TestEnvRule testEnvRule = new TestEnvRule();
+  public static MetricServiceClient metricClient;
+
+  public static String[] VIEWS = {
+    "operation_latencies",
+    "retry_count",
+    "attempt_latencies",
+    "server_latencies",
+    "connectivity_error_count",
+    "application_latencies"
+  };
+
+  @BeforeClass
+  public static void setUpClass() throws IOException {
+    assume()
+        .withMessage("Builtin metrics integration test is not supported by emulator")
+        .that(testEnvRule.env())
+        .isNotInstanceOf(EmulatorEnv.class);
+
+    // Enable built in metrics
+    BigtableDataSettings.enableBuiltinMetrics();
+
+    // Create a cloud monitoring client
+    metricClient = MetricServiceClient.create();
+  }
+
+  @AfterClass
+  public static void tearDown() {
+    if (metricClient != null) {
+      metricClient.close();
+    }
+  }
+
+  @Test
+  public void testBuiltinMetrics() throws Exception {
+    // Send a MutateRow and ReadRows request
+    testEnvRule
+        .env()
+        .getDataClient()
+        .mutateRow(
+            RowMutation.create(testEnvRule.env().getTableId(), "a-new-key")
+                .setCell(testEnvRule.env().getFamilyId(), "q", "abc"));
+    ArrayList<Row> rows =
+        Lists.newArrayList(
+            testEnvRule
+                .env()
+                .getDataClient()
+                .readRows(Query.create(testEnvRule.env().getTableId()).limit(10)));
+
+    // Sleep 5 minutes so the metrics could be published and precomputation is done
+    Thread.sleep(Duration.ofMinutes(5).toMillis());
+
+    ProjectName name = ProjectName.of(testEnvRule.env().getProjectId());
+
+    // Restrict time to last 10 minutes
+    long startMillis = System.currentTimeMillis() - Duration.ofMinutes(10).toMillis();
+    TimeInterval interval =
+        TimeInterval.newBuilder()
+            .setStartTime(Timestamps.fromMillis(startMillis))
+            .setEndTime(Timestamps.fromMillis(System.currentTimeMillis()))
+            .build();
+
+    for (String view : VIEWS) {
+      // Filter on instance and method name
+      // Verify that metrics are published for MutateRow request
+      String metricFilter =
+          String.format(
+              "metric.type=\"bigtable.googleapis.com/client/%s\" "
+                  + "AND resource.labels.instance=\"%s\" AND metric.labels.method=\"Bigtable.MutateRow\"",
+              view, testEnvRule.env().getInstanceId());
+      ListTimeSeriesRequest.Builder requestBuilder =
+          ListTimeSeriesRequest.newBuilder()
+              .setName(name.toString())
+              .setFilter(metricFilter)
+              .setInterval(interval)
+              .setView(ListTimeSeriesRequest.TimeSeriesView.FULL);
+      ListTimeSeriesResponse response =
+          metricClient.listTimeSeriesCallable().call(requestBuilder.build());
+      assertThat(response.getTimeSeriesCount()).isGreaterThan(0);
+
+      // Verify that metrics are published for ReadRows request
+      metricFilter =
+          String.format(
+              "metric.type=\"bigtable.googleapis.com/client/operation_latencies\" "
+                  + "AND resource.labels.instance=\"%s\" AND metric.labels.method=\"Bigtable.ReadRows\"",
+              testEnvRule.env().getInstanceId());
+      requestBuilder.setFilter(metricFilter);
+      response = metricClient.listTimeSeriesCallable().call(requestBuilder.build());
+      assertThat(response.getTimeSeriesCount()).isGreaterThan(0);
+    }
+  }
+}
diff --git a/pom.xml b/pom.xml
index 2b40d71d55..b4c3cfd7b7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -15,6 +15,7 @@
         <groupId>com.google.cloud</groupId>
         <artifactId>google-cloud-shared-config</artifactId>
         <version>1.5.3</version>
+        <relativePath/>
     </parent>
 
     <developers>
@@ -339,6 +340,22 @@
                 </plugins>
             </build>
         </profile>
+
+        <profile>
+            <!-- Shading workaround for IDEs: This project relocates some of its internal dependencies. Since IDEs can't
+             resolve the relocated symbols on their own, a workaround is added: first mvn install the shaded module,
+             then disable the with-shaded profile to force IDEs to resolve the relocated dependencies from the local
+             repository -->
+            <id>with-shaded</id>
+            <activation>
+                <property>
+                    <name>!skip-shaded</name>
+                </property>
+            </activation>
+            <modules>
+                <module>google-cloud-bigtable-stats</module>
+            </modules>
+        </profile>
     </profiles>
 
     <modules>
@@ -347,7 +364,6 @@
     <module>grpc-google-cloud-bigtable-v2</module>
     <module>proto-google-cloud-bigtable-admin-v2</module>
     <module>proto-google-cloud-bigtable-v2</module>
-    <module>google-cloud-bigtable-stats</module>
     <module>google-cloud-bigtable-emulator-core</module>
     <module>google-cloud-bigtable-emulator</module>
     <module>google-cloud-bigtable-bom</module>