From 4c778805f2f9b7bc661e1105a29d89db464d0d52 Mon Sep 17 00:00:00 2001 From: Noah Negrey Date: Thu, 9 Jan 2020 14:33:19 -0700 Subject: [PATCH] docs: add base samples for automl ga (#293) * docs: add base samples for automl ga * update package.json * lint fix * License header update * Fix cleanup on GCS prefix * Lint Co-authored-by: Benjamin E. Coe --- automl/batch_predict.js | 70 +++++++++++++++++++ automl/get_operation_status.js | 53 +++++++++++++++ automl/list_operation_status.js | 52 ++++++++++++++ automl/test/batch_predict.test.js | 83 +++++++++++++++++++++++ automl/test/get_operation_status.test.js | 40 +++++++++++ automl/test/list_operation_status.test.js | 39 +++++++++++ 6 files changed, 337 insertions(+) create mode 100644 automl/batch_predict.js create mode 100644 automl/get_operation_status.js create mode 100644 automl/list_operation_status.js create mode 100644 automl/test/batch_predict.test.js create mode 100644 automl/test/get_operation_status.test.js create mode 100644 automl/test/list_operation_status.test.js diff --git a/automl/batch_predict.js b/automl/batch_predict.js new file mode 100644 index 0000000000..de07dc6af2 --- /dev/null +++ b/automl/batch_predict.js @@ -0,0 +1,70 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +function main( + projectId = 'YOUR_PROJECT_ID', + location = 'us-central1', + modelId = 'YOUR_MODEL_ID', + inputUri = 'gs://YOUR_BUCKET_ID/path_to_your_input_csv_or_jsonl', + outputUri = 'gs://YOUR_BUCKET_ID/path_to_save_results/' +) { + // [START automl_batch_predict] + /** + * TODO(developer): Uncomment these variables before running the sample. + */ + // const projectId = 'YOUR_PROJECT_ID'; + // const location = 'us-central1'; + // const modelId = 'YOUR_MODEL_ID'; + // const inputUri = 'gs://YOUR_BUCKET_ID/path_to_your_input_csv_or_jsonl'; + // const outputUri = 'gs://YOUR_BUCKET_ID/path_to_save_results/'; + + // Imports the Google Cloud AutoML library + const {PredictionServiceClient} = require(`@google-cloud/automl`).v1; + + // Instantiates a client + const client = new PredictionServiceClient(); + + async function batchPredict() { + // Construct request + const request = { + name: client.modelPath(projectId, location, modelId), + inputConfig: { + gcsSource: { + inputUris: [inputUri], + }, + }, + outputConfig: { + gcsDestination: { + outputUriPrefix: outputUri, + }, + }, + }; + + const [operation] = await client.batchPredict(request); + + console.log(`Waiting for operation to complete...`); + // Wait for operation to complete. + const [response] = await operation.promise(); + console.log( + `Batch Prediction results saved to Cloud Storage bucket. ${response}` + ); + } + + batchPredict(); + // [END automl_batch_predict] +} + +main(...process.argv.slice(2)); diff --git a/automl/get_operation_status.js b/automl/get_operation_status.js new file mode 100644 index 0000000000..f11bee08ed --- /dev/null +++ b/automl/get_operation_status.js @@ -0,0 +1,53 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +function main( + projectId = 'YOUR_PROJECT_ID', + location = 'us-central1', + operationId = 'YOUR_OPERATION_ID' +) { + // [START automl_get_operation_status] + /** + * TODO(developer): Uncomment these variables before running the sample. + */ + // const projectId = 'YOUR_PROJECT_ID'; + // const location = 'us-central1'; + // const operationId = 'YOUR_OPERATION_ID'; + + // Imports the Google Cloud AutoML library + const {AutoMlClient} = require(`@google-cloud/automl`).v1; + + // Instantiates a client + const client = new AutoMlClient(); + + async function getOperationStatus() { + // Construct request + const request = { + name: `projects/${projectId}/locations/${location}/operations/${operationId}`, + }; + + const [response] = await client.operationsClient.getOperation(request); + + console.log(`Name: ${response.name}`); + console.log(`Operation details:`); + console.log(`${response}`); + } + + getOperationStatus(); + // [END automl_get_operation_status] +} + +main(...process.argv.slice(2)); diff --git a/automl/list_operation_status.js b/automl/list_operation_status.js new file mode 100644 index 0000000000..233b541358 --- /dev/null +++ b/automl/list_operation_status.js @@ -0,0 +1,52 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +function main(projectId = 'YOUR_PROJECT_ID', location = 'us-central1') { + // [START automl_list_operation_status] + /** + * TODO(developer): Uncomment these variables before running the sample. + */ + // const projectId = 'YOUR_PROJECT_ID'; + // const location = 'us-central1'; + + // Imports the Google Cloud AutoML library + const {AutoMlClient} = require(`@google-cloud/automl`).v1; + + // Instantiates a client + const client = new AutoMlClient(); + + async function listOperationStatus() { + // Construct request + const request = { + name: client.locationPath(projectId, location), + filter: '', + }; + + const [response] = await client.operationsClient.listOperations(request); + + console.log(`List of operation status:`); + for (const operation of response) { + console.log(`Name: ${operation.name}`); + console.log(`Operation details:`); + console.log(`${operation}`); + } + } + + listOperationStatus(); + // [END automl_list_operation_status] +} + +main(...process.argv.slice(2)); diff --git a/automl/test/batch_predict.test.js b/automl/test/batch_predict.test.js new file mode 100644 index 0000000000..b1a16cb4a6 --- /dev/null +++ b/automl/test/batch_predict.test.js @@ -0,0 +1,83 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +const {assert} = require('chai'); +const {after, before, describe, it} = require('mocha'); +const {AutoMlClient} = require('@google-cloud/automl').v1; +const {Storage} = require('@google-cloud/storage'); + +const cp = require('child_process'); + +const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'}); + +const BATCH_PREDICT_REGION_TAG = 'batch_predict'; +const LOCATION = 'us-central1'; +const MODEL_ID = 'TEN2238627664384491520'; +const PREFIX = 'TEST_BATCH_PREDICT'; + +describe('Automl Batch Predict Test', () => { + const client = new AutoMlClient(); + + before('should verify the model is deployed', async () => { + const projectId = await client.getProjectId(); + const request = { + name: client.modelPath(projectId, LOCATION, MODEL_ID), + }; + + const [response] = await client.getModel(request); + if (response.deploymentState === 'UNDEPLOYED') { + const request = { + name: client.modelPath(projectId, LOCATION, MODEL_ID), + }; + + const [operation] = await client.deployModel(request); + + // Wait for operation to complete. + await operation.promise(); + } + }); + + it('should batch predict', async () => { + const projectId = await client.getProjectId(); + const inputUri = `gs://${projectId}-lcm/entity_extraction/input.jsonl`; + const outputUri = `gs://${projectId}-lcm/${PREFIX}/`; + + const batchPredictOutput = execSync( + `node ${BATCH_PREDICT_REGION_TAG}.js ${projectId} ${LOCATION} ${MODEL_ID} ${inputUri} ${outputUri}` + ); + assert.match( + batchPredictOutput, + /Batch Prediction results saved to Cloud Storage bucket/ + ); + }); + + after('delete created files', async () => { + const projectId = await client.getProjectId(); + const storageClient = new Storage(); + const options = { + prefix: PREFIX, + }; + const [files] = await storageClient + .bucket(`gs://${projectId}-lcm`) + .getFiles(options); + files.forEach(file => { + storageClient + .bucket(`gs://${projectId}-lcm`) + .file(file.name) + .delete(); + }); + }); +}); diff --git a/automl/test/get_operation_status.test.js b/automl/test/get_operation_status.test.js new file mode 100644 index 0000000000..c58e188141 --- /dev/null +++ b/automl/test/get_operation_status.test.js @@ -0,0 +1,40 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +const {assert} = require('chai'); +const {describe, it} = require('mocha'); +const {AutoMlClient} = require('@google-cloud/automl').v1; + +const cp = require('child_process'); + +const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'}); + +const GET_OPERATION_STATUS_REGION_TAG = 'get_operation_status'; +const LOCATION = 'us-central1'; +const OPERATION_ID = 'TRL5980949629938696192'; + +describe('Automl Get Operation Status Tests', () => { + const client = new AutoMlClient(); + + it('should get operation status', async () => { + const projectId = await client.getProjectId(); + + const get_output = execSync( + `node ${GET_OPERATION_STATUS_REGION_TAG}.js ${projectId} ${LOCATION} ${OPERATION_ID}` + ); + assert.match(get_output, /Operation details/); + }); +}); diff --git a/automl/test/list_operation_status.test.js b/automl/test/list_operation_status.test.js new file mode 100644 index 0000000000..efbedf30a3 --- /dev/null +++ b/automl/test/list_operation_status.test.js @@ -0,0 +1,39 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +const {assert} = require('chai'); +const {describe, it} = require('mocha'); +const {AutoMlClient} = require('@google-cloud/automl').v1; + +const cp = require('child_process'); + +const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'}); + +const LIST_OPERATION_STATUS_REGION_TAG = 'list_operation_status'; +const LOCATION = 'us-central1'; + +describe('Automl List Operation Status Tests', () => { + const client = new AutoMlClient(); + + it('should list operation status', async () => { + const projectId = await client.getProjectId(); + + const list_output = execSync( + `node ${LIST_OPERATION_STATUS_REGION_TAG}.js ${projectId} ${LOCATION} ` + ); + assert.match(list_output, /Operation details/); + }); +});