From 5aab7af59e46c02e5d8631c3b939c51573df955f Mon Sep 17 00:00:00 2001 From: Dane Zeke Liergaard Date: Tue, 23 Jan 2018 12:53:31 -0800 Subject: [PATCH] Adding BigQuery + StackDriver Monitoring sample. --- appengine-java8/bigquery/README.md | 60 ++++ appengine-java8/bigquery/pom.xml | 137 +++++++++ .../bigquery_logging/BigQueryHome.java | 62 ++++ .../bigquery_logging/BigQueryRun.java | 45 +++ .../bigquery_logging/BigQueryRunner.java | 272 ++++++++++++++++++ .../bigquery_logging/TimeSeriesSummary.java | 99 +++++++ .../src/main/webapp/WEB-INF/appengine-web.xml | 5 + .../bigquery/src/main/webapp/index.jsp | 30 ++ .../bigquery_logging/BigQueryRunnerIT.java | 123 ++++++++ appengine-java8/pom.xml | 2 + 10 files changed, 835 insertions(+) create mode 100644 appengine-java8/bigquery/README.md create mode 100644 appengine-java8/bigquery/pom.xml create mode 100644 appengine-java8/bigquery/src/main/java/com/example/appengine/bigquery_logging/BigQueryHome.java create mode 100644 appengine-java8/bigquery/src/main/java/com/example/appengine/bigquery_logging/BigQueryRun.java create mode 100644 appengine-java8/bigquery/src/main/java/com/example/appengine/bigquery_logging/BigQueryRunner.java create mode 100644 appengine-java8/bigquery/src/main/java/com/example/appengine/bigquery_logging/TimeSeriesSummary.java create mode 100644 appengine-java8/bigquery/src/main/webapp/WEB-INF/appengine-web.xml create mode 100644 appengine-java8/bigquery/src/main/webapp/index.jsp create mode 100644 appengine-java8/bigquery/src/test/java/com/example/appengine/bigquery_logging/BigQueryRunnerIT.java diff --git a/appengine-java8/bigquery/README.md b/appengine-java8/bigquery/README.md new file mode 100644 index 00000000000..f8aca259670 --- /dev/null +++ b/appengine-java8/bigquery/README.md @@ -0,0 +1,60 @@ +# Google Cloud API Showcase: Cloud BigQuery & StackDriver Monitoring in App Engine Standard Java 8 Environment + +This API Showcase demonstrates how to run an AppEngine Standard application with dependencies on both +[Google BigQuery][bigquery] and [StackDriver Monitoring][stackdriver]. + +[bigquery]: https://cloud.google.com/bigquery/docs +[stackdriver]: https://cloud.google.com/monitoring/docs + +The home page of this application provides a form to initiate a query of public data, in this case StackOverflow +questions tagged with `google-bigquery`. + +The home page also provides a summary view of the metrics that have been logged in the past 30 days. + +## Clone the sample app + +Copy the sample apps to your local machine, and cd to the appengine-java8/bigquery directory: + +``` +git clone https://github.com/GoogleCloudPlatform/java-docs-samples +cd appengine-java8/bigquery +``` + +## Setup + +- Make sure [`gcloud`](https://cloud.google.com/sdk/docs/) is installed and initialized: +``` + gcloud init +``` +- If this is the first time you are creating an App Engine project +``` + gcloud app create +``` +- For local development, [set up][set-up] authentication +- Enable [BigQuery][bigquery-api] and [Monitoring][monitoring-api] APIs +- If you have not already enabled your project for StackDriver, do so by following [these instructions][stackdriver-setup]. + +[set-up]: https://cloud.google.com/docs/authentication/getting-started +[bigquery-api]: https://console.cloud.google.com/launcher/details/google/bigquery-json.googleapis.com +[monitoring-api]: https://console.cloud.google.com/launcher/details/google/monitoring.googleapis.com +[stackdriver-setup]: https://cloud.google.com/monitoring/accounts/tiers#not-enabled + +## Run locally +Run using shown Maven command. You can then direct your browser to `http://localhost:8080/` to see the most recent query +run (since the app started) and the metrics from the past 30 days. + +``` +mvn appengine:run +``` + +## Deploy + +- Deploy to AppEngine Standard using the following Maven command. +``` + mvn appengine:deploy +``` +- Direct your browser to `https://.appspot.com`. +- View more in-depth metrics data on the [StackDriver Monitoring Dashboard][dashboard] + +[dashboard]: https://pantheon.corp.google.com/monitoring + diff --git a/appengine-java8/bigquery/pom.xml b/appengine-java8/bigquery/pom.xml new file mode 100644 index 00000000000..5a36a61238d --- /dev/null +++ b/appengine-java8/bigquery/pom.xml @@ -0,0 +1,137 @@ + + + 4.0.0 + war + 1.0-SNAPSHOT + com.example.appengine + appengine-bigquery-monitoring-j8 + + + + com.google.cloud.samples + shared-configuration + 1.0.8 + + + + 1.8 + 1.8 + + + + + com.google.appengine + appengine-api-1.0-sdk + 1.9.60 + + + javax.servlet + javax.servlet-api + 3.1.0 + jar + provided + + + + + com.google.cloud + google-cloud-bigquery + 0.33.0-beta + + + com.google.cloud + google-cloud-monitoring + 0.33.0-beta + + + + + commons-cli + commons-cli + 1.4 + + + joda-time + joda-time + 2.9.9 + + + + + com.google.appengine + appengine-api-stubs + 1.9.60 + test + + + com.google.appengine + appengine-tools-sdk + 1.9.60 + test + + + + junit + junit + 4.12 + test + + + org.mockito + mockito-core + 2.13.0 + test + + + com.google.appengine + appengine-testing + 1.9.60 + test + + + com.google.truth + truth + 0.39 + test + + + + + ${project.build.directory}/${project.build.finalName}/WEB-INF/classes + + + com.google.cloud.tools + appengine-maven-plugin + 1.3.1 + + true + true + + + + + org.apache.maven.plugins + maven-war-plugin + 3.1.0 + + + + + diff --git a/appengine-java8/bigquery/src/main/java/com/example/appengine/bigquery_logging/BigQueryHome.java b/appengine-java8/bigquery/src/main/java/com/example/appengine/bigquery_logging/BigQueryHome.java new file mode 100644 index 00000000000..8b8e696038a --- /dev/null +++ b/appengine-java8/bigquery/src/main/java/com/example/appengine/bigquery_logging/BigQueryHome.java @@ -0,0 +1,62 @@ +package com.example.appengine.bigquery_logging; + +import com.google.cloud.bigquery.FieldValueList; +import com.google.cloud.bigquery.TableResult; +import com.google.protobuf.util.Timestamps; + +import java.io.IOException; +import java.util.List; + +public class BigQueryHome { + private static BigQueryRunner queryRunner; + + private static BigQueryRunner getQueryRunner() throws IOException { + if (queryRunner == null) { + queryRunner = BigQueryRunner.getInstance(); + } + return queryRunner; + } + + public static String getMostRecentRun() throws IOException { + return convertRunToHtmlTable(getQueryRunner().getMostRecentRunResult()); + } + + public static String getMetricAverages() throws IOException { + return convertAveragesToHtmlTable(getQueryRunner().getTimeSeriesValues()); + } + + private static String convertRunToHtmlTable(TableResult result) { + if (result == null) { + return ""; + } + + StringBuilder sb = new StringBuilder(); + for (FieldValueList row : result.iterateAll()) { + sb.append(""); + String url = row.get("url").getStringValue(); + addColumn(sb, String.format("%s", url, url)); + addColumn(sb, row.get("view_count").getLongValue()); + sb.append(""); + } + return sb.toString(); + } + + private static String convertAveragesToHtmlTable(List values) { + + StringBuilder sb = new StringBuilder(); + for(TimeSeriesSummary metric : values) { + sb.append(""); + addColumn(sb, metric.getName()); + addColumn(sb, metric.getValues().size()); + addColumn(sb, metric.getMostRecentRunTime()); + addColumn(sb, metric.getMostRecentValue()); + addColumn(sb, metric.getAverage()); + sb.append(""); + } + return sb.toString(); + } + + private static void addColumn(StringBuilder sb, T content) { + sb.append("").append(content.toString()).append(""); + } +} diff --git a/appengine-java8/bigquery/src/main/java/com/example/appengine/bigquery_logging/BigQueryRun.java b/appengine-java8/bigquery/src/main/java/com/example/appengine/bigquery_logging/BigQueryRun.java new file mode 100644 index 00000000000..4e966323e38 --- /dev/null +++ b/appengine-java8/bigquery/src/main/java/com/example/appengine/bigquery_logging/BigQueryRun.java @@ -0,0 +1,45 @@ +/* + * Copyright 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.appengine.bigquery_logging; + +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +@WebServlet(name = "Run BigQuery", value = "/bigquery/run") +public class BigQueryRun extends HttpServlet { + private BigQueryRunner queryRunner; + + public BigQueryRun() throws IOException { + this.queryRunner = BigQueryRunner.getInstance(); + } + + @Override + public void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOException { + BigQueryRunner queryRunner = this.queryRunner; + try { + queryRunner.Run(); + } catch (InterruptedException e) { + resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, + "Interrupted while running BigQuery job."); + } + // redirect to home page + resp.sendRedirect("/"); + } +} diff --git a/appengine-java8/bigquery/src/main/java/com/example/appengine/bigquery_logging/BigQueryRunner.java b/appengine-java8/bigquery/src/main/java/com/example/appengine/bigquery_logging/BigQueryRunner.java new file mode 100644 index 00000000000..c2d50125372 --- /dev/null +++ b/appengine-java8/bigquery/src/main/java/com/example/appengine/bigquery_logging/BigQueryRunner.java @@ -0,0 +1,272 @@ +/* + * Copyright 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.appengine.bigquery_logging; + +// [START bigquery_logging_simple_app_all] +// [START bigquery_logging_simple_app_deps] + +import com.google.api.Metric; +import com.google.api.MetricDescriptor; +import com.google.cloud.ServiceOptions; +import com.google.cloud.bigquery.*; +import com.google.cloud.monitoring.v3.MetricServiceClient; +import com.google.cloud.monitoring.v3.PagedResponseWrappers.ListMetricDescriptorsPagedResponse; +import com.google.cloud.monitoring.v3.PagedResponseWrappers.ListTimeSeriesPagedResponse; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.monitoring.v3.*; +import com.google.protobuf.util.Timestamps; +import org.joda.time.DateTime; +import org.joda.time.Duration; + +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; +// [END bigquery_logging_simple_app_deps] + +public class BigQueryRunner { + private static final String CUSTOM_METRIC_FILTER = + "metric.type = starts_with(\"custom.googleapis.com/\")"; + private static BigQueryRunner instance; + + // [START bigquery_logging_simple_app_metrics] + private static final MetricDescriptor QUERY_DURATION_METRIC = MetricDescriptor + .newBuilder() + .setName("custom.googleapis.com/queryDuration") + .setType("custom.googleapis.com/queryDuration") + .setDisplayName("queryDuration") + .setDescription("Time it took a query to run.") + .setMetricKind(MetricDescriptor.MetricKind.GAUGE) + .setValueType(MetricDescriptor.ValueType.INT64) + .build(); + private static final MetricDescriptor ROWS_RETURNED_METRIC = MetricDescriptor + .newBuilder() + .setName("custom.googleapis.com/rowsReturned") + .setType("custom.googleapis.com/rowsReturned") + .setDisplayName("rowsReturned") + .setDescription("Total rows returned by the query result.") + .setMetricKind(MetricDescriptor.MetricKind.GAUGE) + .setValueType(MetricDescriptor.ValueType.INT64) + .build(); + private static final Set REQUIRED_METRICS = ImmutableSet.of( + QUERY_DURATION_METRIC, ROWS_RETURNED_METRIC + ); + // [END bigquery_logging_simple_app_metrics] + private TableResult mostRecentRunResult; + private final MetricServiceClient client; + private final String projectName; + private PrintStream os; + private Set existingMetrics = Sets.newHashSet(); + + // Retrieve a singleton instance + public static synchronized BigQueryRunner getInstance() throws IOException { + if (instance == null) { + instance = new BigQueryRunner(); + } + return instance; + } + + public BigQueryRunner() throws IOException { + this(MetricServiceClient.create(), System.out); + } + + public BigQueryRunner(MetricServiceClient metricsClient, PrintStream os) { + client = metricsClient; + this.os = os; + this.projectName = String.format("projects/%s", ServiceOptions.getDefaultProjectId()); + } + + public void Run() throws InterruptedException { + // [START bigquery_logging_simple_app_client] + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + // [END bigquery_logging_simple_app_client] + // [START bigquery_logging_simple_app_query] + QueryJobConfiguration queryConfig = + QueryJobConfiguration.newBuilder( + "SELECT " + + "CONCAT('https://stackoverflow.com/questions/', CAST(id as STRING)) as url, " + + "view_count " + + "FROM `bigquery-public-data.stackoverflow.posts_questions` " + + "WHERE tags like '%google-bigquery%' " + + "ORDER BY favorite_count DESC LIMIT 10") + // Use standard SQL syntax for queries. + // See: https://cloud.google.com/bigquery/sql-reference/ + .setUseLegacySql(false) + .build(); + + List timeSeriesList = new ArrayList<>(); + + DateTime queryStartTime = DateTime.now(); + + // Create a job ID so that we can safely retry. + JobId jobId = JobId.of(UUID.randomUUID().toString()); + Job queryJob = bigquery.create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build()); + + // Wait for the query to complete. + queryJob = queryJob.waitFor(); + + // Check for errors + if (queryJob == null) { + throw new RuntimeException("Job no longer exists"); + } else if (queryJob.getStatus().getError() != null) { + // You can also look at queryJob.getStatus().getExecutionErrors() for all + // errors, not just the latest one. + throw new RuntimeException(queryJob.getStatus().getError().toString()); + } + // [END bigquery_logging_simple_app_query] + + // [START bigquery_logging_simple_app_logmetrics] + // Log the result metrics. + TableResult result = queryJob.getQueryResults(); + + DateTime queryEndTime = DateTime.now(); + // Add query duration metric. + Duration duration = new Duration(queryStartTime, queryEndTime); + timeSeriesList.add(prepareMetric(QUERY_DURATION_METRIC, duration.getMillis())); + + // Add rows returned metric. + timeSeriesList.add(prepareMetric(ROWS_RETURNED_METRIC, result.getTotalRows())); + + // [START bigquery_logging_simple_app_print] + // Print all pages of the results. + for (FieldValueList row : result.iterateAll()) { + String url = row.get("url").getStringValue(); + long viewCount = row.get("view_count").getLongValue(); + os.printf("url: %s views: %d%n", url, viewCount); + } + // [END bigquery_logging_simple_app_print] + + // Prepares the time series request + CreateTimeSeriesRequest request = CreateTimeSeriesRequest.newBuilder() + .setName(projectName) + .addAllTimeSeries(timeSeriesList) + .build(); + + createMetricsIfNeeded(); + client.createTimeSeries(request); + os.println("Done writing metrics."); + // [END bigquery_logging_simple_app_logmetrics] + + mostRecentRunResult = result; + } + + // [START bigquery_logging_get_time_series] + public List getTimeSeriesValues() { + List summaries = Lists.newArrayList(); + createMetricsIfNeeded(); + for (MetricDescriptor metric : REQUIRED_METRICS) { + String format = String.format("metric.type = \"%s\"", metric.getType()); + ListTimeSeriesRequest listTimeSeriesRequest = ListTimeSeriesRequest + .newBuilder() + .setName(projectName) + .setFilter(format) + .setInterval(TimeInterval.newBuilder() + .setStartTime(Timestamps.subtract(Timestamps.fromMillis(System.currentTimeMillis()), + com.google.protobuf.Duration.newBuilder() + .setSeconds(60L * 60L * 24L * 30L) + .build())) + .setEndTime(Timestamps.fromMillis(System.currentTimeMillis())) + .build()) + .build(); + ListTimeSeriesPagedResponse listTimeSeriesResponse = client.listTimeSeries( + listTimeSeriesRequest); + ArrayList timeSeries = Lists.newArrayList(listTimeSeriesResponse.iterateAll()); + summaries.addAll(timeSeries + .stream() + .map(TimeSeriesSummary::fromTimeSeries) + .collect(Collectors.toList())); + } + return summaries; + } + // [END bigquery_logging_get_time_series] + + public TableResult getMostRecentRunResult() { + return mostRecentRunResult; + } + + + // Returns a metric time series with a single int64 data point. + private TimeSeries prepareMetric(MetricDescriptor requiredMetric, long metricValue) { + TimeInterval interval = TimeInterval.newBuilder() + .setEndTime(Timestamps.fromMillis(System.currentTimeMillis())) + .build(); + TypedValue value = TypedValue + .newBuilder() + .setInt64Value(metricValue) + .build(); + + Point point = Point.newBuilder() + .setInterval(interval) + .setValue(value) + .build(); + + List pointList = Lists.newArrayList(); + pointList.add(point); + + Metric metric = Metric.newBuilder() + .setType(requiredMetric.getName()) + .build(); + + return TimeSeries.newBuilder() + .setMetric(metric) + .addAllPoints(pointList) + .build(); + } + + // [START bigquery_logging_list_and_create_metrics] + private void createMetricsIfNeeded() { + // If all required metrics already exist, no need to make service calls. + if (REQUIRED_METRICS.stream() + .map(MetricDescriptor::getDisplayName) + .allMatch(existingMetrics::contains)) { + return; + } + ListMetricDescriptorsRequest listMetricsRequest = ListMetricDescriptorsRequest + .newBuilder() + .setName(projectName) + .setFilter(CUSTOM_METRIC_FILTER) + .build(); + ListMetricDescriptorsPagedResponse listMetricsResponse = client.listMetricDescriptors( + listMetricsRequest); + + for (MetricDescriptor existingMetric : listMetricsResponse.iterateAll()) { + existingMetrics.add(existingMetric.getDisplayName()); + } + + REQUIRED_METRICS.stream() + .filter(metric -> !existingMetrics.contains(metric.getDisplayName())) + .forEach(this::createMetric); + } + // [END bigquery_logging_list_and_create_metrics] + + // [START bigquery_logging_create_metric] + private void createMetric(MetricDescriptor newMetric) { + CreateMetricDescriptorRequest request = CreateMetricDescriptorRequest.newBuilder() + .setName(projectName) + .setMetricDescriptor(newMetric) + .build(); + + existingMetrics.add(client.createMetricDescriptor(request).getDisplayName()); + } +// [END bigquery_logging_create_metric] +} +// [END bigquery_logging_simple_app_all] diff --git a/appengine-java8/bigquery/src/main/java/com/example/appengine/bigquery_logging/TimeSeriesSummary.java b/appengine-java8/bigquery/src/main/java/com/example/appengine/bigquery_logging/TimeSeriesSummary.java new file mode 100644 index 00000000000..8f8d76687e0 --- /dev/null +++ b/appengine-java8/bigquery/src/main/java/com/example/appengine/bigquery_logging/TimeSeriesSummary.java @@ -0,0 +1,99 @@ +package com.example.appengine.bigquery_logging; + +import com.google.common.collect.Collections2; +import com.google.common.collect.Lists; +import com.google.monitoring.v3.Point; +import com.google.monitoring.v3.TimeSeries; +import com.google.protobuf.Timestamp; + +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +public abstract class TimeSeriesSummary { + private String name; + private Timestamp mostRecentRunTime; + T mostRecentValue; + List values; + + public static TimeSeriesSummary fromTimeSeries(TimeSeries timeSeries) { + switch(timeSeries.getValueType()) { + case STRING: + return new StringTimeSeriesSummary(timeSeries); + case INT64: + return new Int64TimeSeriesSummary(timeSeries); + default: + return null; + } + } + + private TimeSeriesSummary(TimeSeries timeSeries) { + name = timeSeries.getMetric().getType(); + } + + Point getMostRecentPoint(TimeSeries timeSeries) { + Point max = Collections.max(timeSeries.getPointsList(), + Comparator.comparingLong(p -> p.getInterval().getEndTime().getSeconds())); + mostRecentRunTime = max.getInterval().getEndTime(); + return max; + } + + public String getName() { + return name; + } + + public T getMostRecentValue() { + return mostRecentValue; + } + + public Timestamp getMostRecentRunTime() { + return mostRecentRunTime; + } + + public List getValues() { + return values; + } + + public abstract T getAverage(); + + public static class StringTimeSeriesSummary extends TimeSeriesSummary { + private StringTimeSeriesSummary(TimeSeries timeSeries) { + super(timeSeries); + Point max = getMostRecentPoint(timeSeries); + if (max == null) { + return; + } + mostRecentValue = max + .getValue() + .getStringValue(); + values = Lists.newArrayList(Collections2.transform(timeSeries.getPointsList(), + point -> point.getValue().getStringValue())); + } + + @Override + public String getAverage() { + return values.stream().collect(Collectors.joining(",")); + } + } + + public static class Int64TimeSeriesSummary extends TimeSeriesSummary { + private Int64TimeSeriesSummary(TimeSeries timeSeries) { + super(timeSeries); + Point max = getMostRecentPoint(timeSeries); + if (max == null) { + return; + } + mostRecentValue = max + .getValue() + .getInt64Value(); + values = Lists.newArrayList(Collections2.transform(timeSeries.getPointsList(), + point -> point.getValue().getInt64Value())); + } + + @Override + public Long getAverage() { + return values.stream().collect(Collectors.averagingLong(Long::longValue)).longValue(); + } + } +} diff --git a/appengine-java8/bigquery/src/main/webapp/WEB-INF/appengine-web.xml b/appengine-java8/bigquery/src/main/webapp/WEB-INF/appengine-web.xml new file mode 100644 index 00000000000..0b91ca02261 --- /dev/null +++ b/appengine-java8/bigquery/src/main/webapp/WEB-INF/appengine-web.xml @@ -0,0 +1,5 @@ + + true + java8 + dzlier-work + diff --git a/appengine-java8/bigquery/src/main/webapp/index.jsp b/appengine-java8/bigquery/src/main/webapp/index.jsp new file mode 100644 index 00000000000..d32527a908f --- /dev/null +++ b/appengine-java8/bigquery/src/main/webapp/index.jsp @@ -0,0 +1,30 @@ +<%@ page import="com.example.appengine.bigquery_logging.BigQueryHome" %> + + + An example of using BigQuery and StackDriver Logging on AppEngine Standard + +

Run query

+
+ +
+

Most Recent Run Results

+ + + + + + <%= BigQueryHome.getMostRecentRun() %> +
URLView Count
+

Run Metric Values

+ + + + + + + + + <%= BigQueryHome.getMetricAverages() %> +
Metric TypeCountMost Recent End TimeMost Recent ValueAverage/Values
+ + diff --git a/appengine-java8/bigquery/src/test/java/com/example/appengine/bigquery_logging/BigQueryRunnerIT.java b/appengine-java8/bigquery/src/test/java/com/example/appengine/bigquery_logging/BigQueryRunnerIT.java new file mode 100644 index 00000000000..417adf7a65b --- /dev/null +++ b/appengine-java8/bigquery/src/test/java/com/example/appengine/bigquery_logging/BigQueryRunnerIT.java @@ -0,0 +1,123 @@ +/* + * Copyright 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.appengine.bigquery_logging; + +import com.google.api.MetricDescriptor; +import com.google.api.gax.rpc.UnaryCallable; +import com.google.cloud.monitoring.v3.MetricServiceClient; +import com.google.cloud.monitoring.v3.PagedResponseWrappers.ListMetricDescriptorsPagedResponse; +import com.google.cloud.monitoring.v3.stub.MetricServiceStub; +import com.google.monitoring.v3.CreateMetricDescriptorRequest; +import com.google.monitoring.v3.CreateTimeSeriesRequest; +import com.google.monitoring.v3.ListMetricDescriptorsRequest; +import com.google.protobuf.Empty; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.util.Collections; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +/** + * Tests for simple app sample. + */ +@RunWith(JUnit4.class) +@SuppressWarnings("checkstyle:abbreviationaswordinname") +public class BigQueryRunnerIT { + private static final String LEGACY_PROJECT_ENV_NAME = "GCLOUD_PROJECT"; + private static final String PROJECT_ENV_NAME = "GOOGLE_CLOUD_PROJECT"; + + private ByteArrayOutputStream bout; + private BigQueryRunner app; + + @Mock + private MetricServiceStub metricsServiceStub; + @Mock + private UnaryCallable listCallable; + @Mock + private UnaryCallable createMetricCallable; + @Mock + private UnaryCallable createTimeSeriesCallable; + @Mock + private ListMetricDescriptorsPagedResponse listResponse; + + @Captor + private ArgumentCaptor createTimeSeriesRequest; + + private static String getProjectId() { + String projectId = System.getProperty(PROJECT_ENV_NAME, System.getenv(PROJECT_ENV_NAME)); + if (projectId == null) { + projectId = System.getProperty(LEGACY_PROJECT_ENV_NAME, + System.getenv(LEGACY_PROJECT_ENV_NAME)); + } + return projectId; + } + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + bout = new ByteArrayOutputStream(); + PrintStream out = new PrintStream(bout); + + MetricServiceClient metricsClient = MetricServiceClient.create(metricsServiceStub); + System.setProperty("projectId", BigQueryRunnerIT.getProjectId()); + app = new BigQueryRunner(metricsClient, out); + + when(metricsServiceStub.listMetricDescriptorsPagedCallable()).thenReturn(listCallable); + when(listCallable.call(any(ListMetricDescriptorsRequest.class))).thenReturn(listResponse); + when(listResponse.iterateAll()).thenReturn(Collections.EMPTY_LIST); + + when(metricsServiceStub.createMetricDescriptorCallable()).thenReturn(createMetricCallable); + when(createMetricCallable.call(any(CreateMetricDescriptorRequest.class))).thenReturn(null); + + when(metricsServiceStub.createTimeSeriesCallable()).thenReturn(createTimeSeriesCallable); + when(createTimeSeriesCallable.call(any(CreateTimeSeriesRequest.class))) + .thenReturn(Empty.getDefaultInstance()); + } + + @Test + public void testRun() throws Exception { + app.Run(); + String got = bout.toString(); + assertThat(got).contains("views:"); + verify(metricsServiceStub).listMetricDescriptorsPagedCallable(); + + verify(metricsServiceStub, times(2)).createMetricDescriptorCallable(); + + verify(metricsServiceStub).createTimeSeriesCallable(); + verify(createTimeSeriesCallable).call(createTimeSeriesRequest.capture()); + CreateTimeSeriesRequest actual = createTimeSeriesRequest.getValue(); + assertEquals(2, actual.getTimeSeriesCount()); + assertThat(actual.getTimeSeries(0).getMetric().getType()).isEqualTo( + "custom.googleapis.com/queryDuration"); + assertThat(actual.getTimeSeries(0).getPoints(0).getValue().getInt64Value()).isGreaterThan(0L); + assertThat(actual.getTimeSeries(1).getMetric().getType()).isEqualTo( + "custom.googleapis.com/rowsReturned"); + assertThat(actual.getTimeSeries(1).getPoints(0).getValue().getInt64Value()).isGreaterThan(0L); + } +} diff --git a/appengine-java8/pom.xml b/appengine-java8/pom.xml index 3b3b23e92d4..8b96e8d923d 100644 --- a/appengine-java8/pom.xml +++ b/appengine-java8/pom.xml @@ -39,6 +39,7 @@ analytics appidentity + bigquery bigtable cloudsql cloudsql-postgres @@ -74,6 +75,7 @@ taskqueues-deferred taskqueues-pull taskqueues-push + translate-pubsub twilio urlfetch users