From f1809d6cedd7f547029084d12443ea032b8ea1be Mon Sep 17 00:00:00 2001 From: dzlier Date: Mon, 22 Jan 2018 09:33:25 -0800 Subject: [PATCH] Adding BigQuery + StackDriver Monitoring API Showcase sample. --- bigquery/cloud-client/README.md | 42 ++++ bigquery/cloud-client/pom.xml | 13 +- .../bigquery_logging/RequiredMetric.java | 86 +++++++ .../example/bigquery_logging/SimpleApp.java | 229 ++++++++++++++++++ 4 files changed, 368 insertions(+), 2 deletions(-) create mode 100644 bigquery/cloud-client/src/main/java/com/example/bigquery_logging/RequiredMetric.java create mode 100644 bigquery/cloud-client/src/main/java/com/example/bigquery_logging/SimpleApp.java diff --git a/bigquery/cloud-client/README.md b/bigquery/cloud-client/README.md index a6e15af880f..df1b8cdfd30 100644 --- a/bigquery/cloud-client/README.md +++ b/bigquery/cloud-client/README.md @@ -46,3 +46,45 @@ documentation](https://cloud.google.com/bigquery/create-simple-app-api): mvn exec:java -Dexec.mainClass=com.example.bigquery.SimpleApp +## BigQuery + Logging SimpleApp + +This API Showcase demonstrates how to run a BigQuery query and then log some metrics to StackDriver Monitoring +from the results, including the query runtime and number of rows returned. + +### Clone the sample app + +Copy the sample apps to your local machine, and cd to the bigquery/cloud-client directory: + +``` +git clone https://github.com/GoogleCloudPlatform/java-docs-samples +cd java-docs-samples/bigquery/cloud-client +``` + +### Setup + +- Make sure [`gcloud`](https://cloud.google.com/sdk/docs/) is installed and initialized: +``` + gcloud init +``` +- If this is the first time you are creating an App Engine project +``` + gcloud app create +``` +- For local development, [set up][set-up] authentication +- Enable [BigQuery][bigquery-api] and [Monitoring][monitoring-api] APIs +- If you have not already enabled your project for StackDriver, do so by following [these instructions][stackdriver-setup]. + +[set-up]: https://cloud.google.com/docs/authentication/getting-started +[bigquery-api]: https://console.cloud.google.com/launcher/details/google/bigquery-json.googleapis.com +[monitoring-api]: https://console.cloud.google.com/launcher/details/google/monitoring.googleapis.com +[stackdriver-setup]: https://cloud.google.com/monitoring/accounts/tiers#not-enabled + +### Run the SimpleApp + +Build your project with: + + mvn clean package -DskipTests + +Provide the projectId as a system variable when running the sample. + + mvn exec:java -Dexec.mainClass=com.example.bigquery_logging.SimpleApp -DprojectId= \ No newline at end of file diff --git a/bigquery/cloud-client/pom.xml b/bigquery/cloud-client/pom.xml index 653b6aa8766..0b0f395518a 100644 --- a/bigquery/cloud-client/pom.xml +++ b/bigquery/cloud-client/pom.xml @@ -30,8 +30,8 @@ - 1.7 - 1.7 + 1.8 + 1.8 UTF-8 @@ -43,6 +43,15 @@ 0.33.0-beta + + + + com.google.cloud + google-cloud-monitoring + 0.33.0-beta + + + commons-cli commons-cli diff --git a/bigquery/cloud-client/src/main/java/com/example/bigquery_logging/RequiredMetric.java b/bigquery/cloud-client/src/main/java/com/example/bigquery_logging/RequiredMetric.java new file mode 100644 index 00000000000..f542667824f --- /dev/null +++ b/bigquery/cloud-client/src/main/java/com/example/bigquery_logging/RequiredMetric.java @@ -0,0 +1,86 @@ +/* + * Copyright 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigquery_logging; + +import com.google.api.MetricDescriptor; + +import java.util.Objects; + +public class RequiredMetric { + private String shortName; + private String name; + private String description; + private MetricDescriptor.MetricKind metricKind; + private MetricDescriptor.ValueType valueType; + + public RequiredMetric(String shortName, String description) { + this.shortName = shortName; + this.name = "custom.googleapis.com/" + shortName; + this.description = description; + this.metricKind = MetricDescriptor.MetricKind.GAUGE; + this.valueType = MetricDescriptor.ValueType.INT64; + } + + public RequiredMetric(String shortName, + String name, + String description, + MetricDescriptor.MetricKind metricKind, + MetricDescriptor.ValueType valueType) { + this.shortName = shortName; + this.name = name; + this.description = description; + this.metricKind = metricKind; + this.valueType = valueType; + } + + public String getShortName() { + return shortName; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public MetricDescriptor.MetricKind getMetricKind() { + return metricKind; + } + + public MetricDescriptor.ValueType getValueType() { + return valueType; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RequiredMetric that = (RequiredMetric) o; + return Objects.equals(name, that.name) && + metricKind == that.metricKind && + valueType == that.valueType; + } + + @Override + public int hashCode() { + return Objects.hash(name, metricKind, valueType); + } + + +} diff --git a/bigquery/cloud-client/src/main/java/com/example/bigquery_logging/SimpleApp.java b/bigquery/cloud-client/src/main/java/com/example/bigquery_logging/SimpleApp.java new file mode 100644 index 00000000000..01feb26e952 --- /dev/null +++ b/bigquery/cloud-client/src/main/java/com/example/bigquery_logging/SimpleApp.java @@ -0,0 +1,229 @@ +/* + * Copyright 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.bigquery_logging; + +// [START bigquery_logging_simple_app_all] +// [START bigquery_logging_simple_app_deps] + +import com.google.api.Metric; +import com.google.api.MetricDescriptor; +import com.google.cloud.bigquery.*; +import com.google.cloud.monitoring.v3.MetricServiceClient; +import com.google.cloud.monitoring.v3.PagedResponseWrappers; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.monitoring.v3.*; +import com.google.protobuf.util.Timestamps; +import org.joda.time.DateTime; +import org.joda.time.Duration; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.TimeUnit; +// [END bigquery_logging_simple_app_deps] + +public class SimpleApp { + // [START bigquery_logging_simple_app_metrics] + private static final RequiredMetric QUERY_DURATION_METRIC = new RequiredMetric( + "queryDuration", + "Time it took a query to run."); + private static final RequiredMetric ROWS_RETURNED_METRIC = new RequiredMetric( + "rowsReturned", + "Total rows returned by the query result."); + private static final Set REQUIRED_METRICS = ImmutableSet.of( + QUERY_DURATION_METRIC, + ROWS_RETURNED_METRIC + ); + // [END bigquery_logging_simple_app_metrics] + private final MetricServiceClient client; + private final String projectName; + + private SimpleApp() throws IOException { + this(MetricServiceClient.create()); + } + + private SimpleApp(MetricServiceClient metricsClient) { + client = metricsClient; + projectName = String.format("projects/%s", System.getProperty("projectId")); + } + + public static void main(String... args) throws Exception { + SimpleApp app = new SimpleApp(); + app.Run(); + } + + private void Run() throws InterruptedException { + // [START bigquery_logging_simple_app_client] + BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); + // [END bigquery_logging_simple_app_client] + // [START bigquery_logging_simple_app_query] + QueryJobConfiguration queryConfig = + QueryJobConfiguration.newBuilder( + "SELECT " + + "CONCAT('https://stackoverflow.com/questions/', CAST(id as STRING)) as url, " + + "view_count " + + "FROM `bigquery-public-data.stackoverflow.posts_questions` " + + "WHERE tags like '%google-bigquery%' " + + "ORDER BY favorite_count DESC LIMIT 10") + // Use standard SQL syntax for queries. + // See: https://cloud.google.com/bigquery/sql-reference/ + .setUseLegacySql(false) + .build(); + + List timeSeriesList = new ArrayList<>(); + + DateTime queryStartTime = DateTime.now(); + + // Create a job ID so that we can safely retry. + JobId jobId = JobId.of(UUID.randomUUID().toString()); + Job queryJob = bigquery.create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build()); + + // Wait for the query to complete. + queryJob = queryJob.waitFor(); + + // Check for errors + if (queryJob == null) { + throw new RuntimeException("Job no longer exists"); + } else if (queryJob.getStatus().getError() != null) { + // You can also look at queryJob.getStatus().getExecutionErrors() for all + // errors, not just the latest one. + throw new RuntimeException(queryJob.getStatus().getError().toString()); + } + // [END bigquery_logging_simple_app_query] + + // [START bigquery_logging_simple_app_logmetrics] + // Log the result metrics. + TableResult result = queryJob.getQueryResults(); + + DateTime queryEndTime = DateTime.now(); + // Add query duration metric. + Duration duration = new Duration(queryStartTime, queryEndTime); + timeSeriesList.add(prepareMetric(QUERY_DURATION_METRIC, duration.getMillis())); + + // Add rows returned metric. + timeSeriesList.add(prepareMetric(ROWS_RETURNED_METRIC, result.getTotalRows())); + + // [START bigquery_logging_simple_app_print] + // Print all pages of the results. + for (FieldValueList row : result.iterateAll()) { + String url = row.get("url").getStringValue(); + long viewCount = row.get("view_count").getLongValue(); + System.out.printf("url: %s views: %d%n", url, viewCount); + } + // [END bigquery_logging_simple_app_print] + + // Prepares the time series request + CreateTimeSeriesRequest request = CreateTimeSeriesRequest.newBuilder() + .setName(projectName) + .addAllTimeSeries(timeSeriesList) + .build(); + + createMetricsIfNeeded(); + client.createTimeSeries(request); + System.out.println("Done writing metrics."); + System.out.println("Shutting down MetricsServiceClient."); + try { + if (!client.awaitTermination(5, TimeUnit.SECONDS)) { + client.shutdownNow(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + client.shutdownNow(); + } + // [END bigquery_logging_simple_app_logmetrics] + } + + // Returns a metric time series with a single int64 data point. + private TimeSeries prepareMetric(RequiredMetric requiredMetric, long metricValue) { + TimeInterval interval = TimeInterval.newBuilder() + .setEndTime(Timestamps.fromMillis(System.currentTimeMillis())) + .build(); + TypedValue value = TypedValue + .newBuilder() + .setInt64Value(metricValue) + .build(); + + Point point = Point.newBuilder() + .setInterval(interval) + .setValue(value) + .build(); + + List pointList = Lists.newArrayList(); + pointList.add(point); + + Metric metric = Metric.newBuilder() + .setType(requiredMetric.getName()) + .build(); + + return TimeSeries.newBuilder() + .setMetric(metric) + .addAllPoints(pointList) + .build(); + } + + // [START bigquery_logging_list_and_create_metrics] + private void createMetricsIfNeeded() { + ListMetricDescriptorsRequest listMetricsRequest = ListMetricDescriptorsRequest + .newBuilder() + .setName(projectName) + .setFilter("metric.type = starts_with(\"custom.googleapis.com/\")") + .build(); + PagedResponseWrappers.ListMetricDescriptorsPagedResponse listMetricsResponse = + client.listMetricDescriptors(listMetricsRequest); + + Map existingMetrics = Maps.newHashMap(Maps.asMap(REQUIRED_METRICS, + metric -> false)); + for (MetricDescriptor d : listMetricsResponse.iterateAll()) { + RequiredMetric existingMetric = new RequiredMetric( + d.getDisplayName(), + d.getName(), + d.getDescription(), + d.getMetricKind(), + d.getValueType()); + existingMetrics.put(existingMetric, true); + } + + existingMetrics.forEach((metric, exists) -> { + if (!exists) { + createMetric(metric); + } + }); + } + // [END bigquery_logging_list_and_create_metrics] + + // [START bigquery_logging_create_metric] + private void createMetric(RequiredMetric newMetric) { + MetricDescriptor descriptor = MetricDescriptor.newBuilder() + .setType(newMetric.getName()) + .setName(newMetric.getName()) + .setDisplayName(newMetric.getShortName()) + .setDescription(newMetric.getDescription()) + .setMetricKind(newMetric.getMetricKind()) + .setValueType(newMetric.getValueType()) + .build(); + + CreateMetricDescriptorRequest request = CreateMetricDescriptorRequest.newBuilder() + .setName(projectName) + .setMetricDescriptor(descriptor) + .build(); + + client.createMetricDescriptor(request); + } +// [END bigquery_logging_create_metric] +} +// [END bigquery_logging_simple_app_all]