From 36e13a1fe5ebe09bc9268cdc7b21c6f2882f305a Mon Sep 17 00:00:00 2001 From: Jason Dobry Date: Mon, 24 Apr 2017 15:46:16 -0700 Subject: [PATCH] Bring BigQuery samples up to standard. --- bigquery/README.md | 79 +++--- bigquery/datasets.js | 162 ++++++----- bigquery/package.json | 2 +- bigquery/queries.js | 96 ++++--- bigquery/system-test/datasets.test.js | 22 +- bigquery/system-test/tables.test.js | 8 +- bigquery/tables.js | 395 +++++++++++++++++--------- bigquery/yarn.lock | 6 +- 8 files changed, 452 insertions(+), 318 deletions(-) diff --git a/bigquery/README.md b/bigquery/README.md index 66ab79429e..cf02044f41 100644 --- a/bigquery/README.md +++ b/bigquery/README.md @@ -42,21 +42,21 @@ __Usage:__ `node datasets --help` ``` Commands: - create Creates a new dataset. - delete Deletes a dataset. - list [projectId] Lists all datasets in the specified project or the current project. - size [projectId] Calculates the size of a dataset. + create Creates a new dataset. + delete Deletes a dataset. + list Lists datasets. Options: - --help Show help [boolean] + --projectId, -p The Project ID to use. Defaults to the value of the GCLOUD_PROJECT or GOOGLE_CLOUD_PROJECT + environment variables. [string] + --help Show help [boolean] Examples: - node datasets create my_dataset Creates a new dataset named "my_dataset". - node datasets delete my_dataset Deletes a dataset named "my_dataset". - node datasets list Lists all datasets in the current project. - node datasets list bigquery-public-data Lists all datasets in the "bigquery-public-data" project. - node datasets size my_dataset Calculates the size of "my_dataset" in the current project. - node datasets size hacker_news bigquery-public-data Calculates the size of "bigquery-public-data:hacker_news". + node datasets.js create my_dataset Creates a new dataset named "my_dataset". + node datasets.js delete my_dataset Deletes a dataset named "my_dataset". + node datasets.js list Lists all datasets in the project specified by the + GCLOUD_PROJECT or GOOGLE_CLOUD_PROJECT environments variables. + node datasets.js list --projectId=bigquery-public-data Lists all datasets in the "bigquery-public-data" project. For more information, see https://cloud.google.com/bigquery/docs ``` @@ -77,14 +77,16 @@ Commands: shakespeare Queries a public Shakespeare dataset. Options: - --help Show help [boolean] + --projectId, -p The Project ID to use. Defaults to the value of the GCLOUD_PROJECT or GOOGLE_CLOUD_PROJECT + environment variables. [string] + --help Show help [boolean] Examples: - node queries sync "SELECT * FROM publicdata.samples.natality Synchronously queries the natality dataset. - LIMIT 5;" - node queries async "SELECT * FROM Queries the natality dataset as a job. + node queries.js sync "SELECT * FROM Synchronously queries the natality dataset. publicdata.samples.natality LIMIT 5;" - node queries shakespeare Queries a public Shakespeare dataset. + node queries.js async "SELECT * FROM Queries the natality dataset as a job. + publicdata.samples.natality LIMIT 5;" + node queries.js shakespeare Queries a public Shakespeare dataset. For more information, see https://cloud.google.com/bigquery/docs ``` @@ -100,41 +102,42 @@ __Usage:__ `node tables --help` ``` Commands: - create [projectId] Creates a new table. - list [projectId] Lists all tables in a dataset. - delete [projectId] Deletes a table. + create Creates a new table. + list Lists all tables in a dataset. + delete Deletes a table. copy Makes a copy of a table. - [projectId] - browse [projectId] Lists rows in a table. - import [projectId] Imports data from a local file into a table. + + browse Lists rows in a table. + import Imports data from a local file into a table. import-gcs Imports data from a Google Cloud Storage file into a - [projectId] table. + table. export Export a table from BigQuery to Google Cloud Storage. - [projectId] - insert [projectId] Insert a JSON array (as a string or newline-delimited + insert Insert a JSON array (as a string or newline-delimited file) into a BigQuery table. Options: - --help Show help [boolean] + --projectId, -p The Project ID to use. Defaults to the value of the GCLOUD_PROJECT or GOOGLE_CLOUD_PROJECT + environment variables. [string] + --help Show help [boolean] Examples: - node tables create my_dataset my_table "Name:string, Createss a new table named "my_table" in "my_dataset". + node tables.js create my_dataset my_table "Name:string, Creates a new table named "my_table" in "my_dataset". Age:integer, Weight:float, IsMagic:boolean" - node tables list my_dataset Lists tables in "my_dataset". - node tables browse my_dataset my_table Displays rows from "my_table" in "my_dataset". - node tables delete my_dataset my_table Deletes "my_table" from "my_dataset". - node tables import my_dataset my_table ./data.csv Imports a local file into a table. - node tables import-gcs my_dataset my_table my-bucket Imports a GCS file into a table. + node tables.js list my_dataset Lists tables in "my_dataset". + node tables.js browse my_dataset my_table Displays rows from "my_table" in "my_dataset". + node tables.js delete my_dataset my_table Deletes "my_table" from "my_dataset". + node tables.js import my_dataset my_table ./data.csv Imports a local file into a table. + node tables.js import-gcs my_dataset my_table my-bucket Imports a GCS file into a table. data.csv - node tables export my_dataset my_table my-bucket my-file Exports my_dataset:my_table to gcs://my-bucket/my-file + node tables.js export my_dataset my_table my-bucket my-file Exports my_dataset:my_table to gcs://my-bucket/my-file as raw CSV. - node tables export my_dataset my_table my-bucket my-file -f Exports my_dataset:my_table to gcs://my-bucket/my-file - JSON --gzip as gzipped JSON. - node tables insert my_dataset my_table json_string Inserts the JSON array represented by json_string into + node tables.js export my_dataset my_table my-bucket my-file Exports my_dataset:my_table to gcs://my-bucket/my-file + -f JSON --gzip as gzipped JSON. + node tables.js insert my_dataset my_table json_string Inserts the JSON array represented by json_string into my_dataset:my_table. - node tables insert my_dataset my_table json_file Inserts the JSON objects contained in json_file (one per + node tables.js insert my_dataset my_table json_file Inserts the JSON objects contained in json_file (one per line) into my_dataset:my_table. - node tables copy src_dataset src_table dest_dataset Copies src_dataset:src_table to dest_dataset:dest_table. + node tables.js copy src_dataset src_table dest_dataset Copies src_dataset:src_table to dest_dataset:dest_table. dest_table For more information, see https://cloud.google.com/bigquery/docs diff --git a/bigquery/datasets.js b/bigquery/datasets.js index 944730cc2d..7b543fb0b1 100644 --- a/bigquery/datasets.js +++ b/bigquery/datasets.js @@ -15,123 +15,129 @@ 'use strict'; -const BigQuery = require('@google-cloud/bigquery'); +function createDataset (datasetId, projectId) { + // [START bigquery_create_dataset] + // Imports the Google Cloud client library + const BigQuery = require('@google-cloud/bigquery'); + + // The project ID to use, e.g. "your-project-id" + // const projectId = "your-project-id"; -// [START bigquery_create_dataset] -function createDataset (datasetId) { // Instantiates a client - const bigquery = BigQuery(); + const bigquery = BigQuery({ + projectId: projectId + }); + + // The ID for the new dataset, e.g. "my_new_dataset" + // const datasetId = "my_new_dataset"; - // Creates a new dataset, e.g. "my_new_dataset" - return bigquery.createDataset(datasetId) + // Creates a new dataset + bigquery.createDataset(datasetId) .then((results) => { const dataset = results[0]; console.log(`Dataset ${dataset.id} created.`); - return dataset; + }) + .catch((err) => { + console.error('ERROR:', err); }); + // [END bigquery_create_dataset] } -// [END bigquery_create_dataset] -// [START bigquery_delete_dataset] -function deleteDataset (datasetId) { +function deleteDataset (datasetId, projectId) { + // [START bigquery_delete_dataset] + // Imports the Google Cloud client library + const BigQuery = require('@google-cloud/bigquery'); + + // The project ID to use, e.g. "your-project-id" + // const projectId = "your-project-id"; + // Instantiates a client - const bigquery = BigQuery(); + const bigquery = BigQuery({ + projectId: projectId + }); + + // The ID of the dataset to delete, e.g. "my_new_dataset" + // const datasetId = "my_new_dataset"; - // References an existing dataset, e.g. "my_dataset" + // Creates a reference to the existing dataset const dataset = bigquery.dataset(datasetId); // Deletes the dataset - return dataset.delete() + dataset.delete() .then(() => { console.log(`Dataset ${dataset.id} deleted.`); + }) + .catch((err) => { + console.error('ERROR:', err); }); + // [END bigquery_delete_dataset] } -// [END bigquery_delete_dataset] -// [START bigquery_list_datasets] function listDatasets (projectId) { + // [START bigquery_list_datasets] + // Imports the Google Cloud client library + const BigQuery = require('@google-cloud/bigquery'); + + // The project ID to use, e.g. "your-project-id" + // const projectId = "your-project-id"; + // Instantiates a client const bigquery = BigQuery({ projectId: projectId }); // Lists all datasets in the specified project - return bigquery.getDatasets() + bigquery.getDatasets() .then((results) => { const datasets = results[0]; console.log('Datasets:'); datasets.forEach((dataset) => console.log(dataset.id)); - return datasets; + }) + .catch((err) => { + console.error('ERROR:', err); }); + // [END bigquery_list_datasets] } -// [END bigquery_list_datasets] - -// [START bigquery_get_dataset_size] -function getDatasetSize (datasetId, projectId) { - // Instantiate a client - const bigquery = BigQuery({ - projectId: projectId - }); - - // References an existing dataset, e.g. "my_dataset" - const dataset = bigquery.dataset(datasetId); - // Lists all tables in the dataset - return dataset.getTables() - .then((results) => results[0]) - // Retrieve the metadata for each table - .then((tables) => Promise.all(tables.map((table) => table.get()))) - .then((results) => results.map((result) => result[0])) - // Select the size of each table - .then((tables) => tables.map((table) => (parseInt(table.metadata.numBytes, 10) / 1000) / 1000)) - // Sum up the sizes - .then((sizes) => sizes.reduce((cur, prev) => cur + prev, 0)) - // Print and return the size - .then((sum) => { - console.log(`Size of ${dataset.id}: ${sum} MB`); - return sum; - }); -} -// [END bigquery_get_dataset_size] - -// The command-line program -const cli = require(`yargs`); - -const program = module.exports = { - createDataset: createDataset, - deleteDataset: deleteDataset, - listDatasets: listDatasets, - getDatasetSize: getDatasetSize, - main: (args) => { - // Run the command-line program - cli.help().strict().parse(args).argv; // eslint-disable-line - } -}; - -cli +const cli = require(`yargs`) .demand(1) - .command(`create `, `Creates a new dataset.`, {}, (opts) => { - program.createDataset(opts.datasetId); - }) - .command(`delete `, `Deletes a dataset.`, {}, (opts) => { - program.deleteDataset(opts.datasetId); - }) - .command(`list [projectId]`, `Lists all datasets in the specified project or the current project.`, {}, (opts) => { - program.listDatasets(opts.projectId || process.env.GCLOUD_PROJECT); - }) - .command(`size [projectId]`, `Calculates the size of a dataset.`, {}, (opts) => { - program.getDatasetSize(opts.datasetId, opts.projectId || process.env.GCLOUD_PROJECT); + .options({ + projectId: { + alias: 'p', + default: process.env.GCLOUD_PROJECT || process.env.GOOGLE_CLOUD_PROJECT, + description: 'The Project ID to use. Defaults to the value of the GCLOUD_PROJECT or GOOGLE_CLOUD_PROJECT environment variables.', + requiresArg: true, + type: 'string' + } }) + .command( + `create `, + `Creates a new dataset.`, + {}, + (opts) => createDataset(opts.datasetId, opts.projectId) + ) + .command( + `delete `, + `Deletes a dataset.`, + {}, + (opts) => deleteDataset(opts.datasetId, opts.projectId) + ) + .command( + `list`, + `Lists datasets.`, + {}, + (opts) => listDatasets(opts.projectId) + ) .example(`node $0 create my_dataset`, `Creates a new dataset named "my_dataset".`) .example(`node $0 delete my_dataset`, `Deletes a dataset named "my_dataset".`) - .example(`node $0 list`, `Lists all datasets in the current project.`) - .example(`node $0 list bigquery-public-data`, `Lists all datasets in the "bigquery-public-data" project.`) - .example(`node $0 size my_dataset`, `Calculates the size of "my_dataset" in the current project.`) - .example(`node $0 size hacker_news bigquery-public-data`, `Calculates the size of "bigquery-public-data:hacker_news".`) + .example(`node $0 list`, `Lists all datasets in the project specified by the GCLOUD_PROJECT or GOOGLE_CLOUD_PROJECT environments variables.`) + .example(`node $0 list --projectId=bigquery-public-data`, `Lists all datasets in the "bigquery-public-data" project.`) .wrap(120) .recommendCommands() - .epilogue(`For more information, see https://cloud.google.com/bigquery/docs`); + .epilogue(`For more information, see https://cloud.google.com/bigquery/docs`) + .help() + .strict(); if (module === require.main) { - program.main(process.argv.slice(2)); + cli.parse(process.argv.slice(2)); } diff --git a/bigquery/package.json b/bigquery/package.json index b12ff58196..5684205cf0 100644 --- a/bigquery/package.json +++ b/bigquery/package.json @@ -27,7 +27,7 @@ "yargs": "7.1.0" }, "devDependencies": { - "@google-cloud/nodejs-repo-tools": "1.3.1", + "@google-cloud/nodejs-repo-tools": "1.3.2", "ava": "0.19.1", "proxyquire": "1.7.11", "sinon": "2.1.0", diff --git a/bigquery/queries.js b/bigquery/queries.js index b70eb57234..387416afe6 100644 --- a/bigquery/queries.js +++ b/bigquery/queries.js @@ -16,8 +16,6 @@ 'use strict'; // [START bigquery_simple_app_all] -const BigQuery = require('@google-cloud/bigquery'); - // [START bigquery_simple_app_print] function printResult (rows) { console.log('Query Results:'); @@ -43,9 +41,17 @@ GROUP BY ORDER BY unique_words DESC LIMIT 10;`; -function queryShakespeare () { +function queryShakespeare (projectId) { + // Imports the Google Cloud client library + const BigQuery = require('@google-cloud/bigquery'); + + // The project ID to use, e.g. "your-project-id" + // const projectId = "your-project-id"; + // Instantiates a client - const bigquery = BigQuery(); + const bigquery = BigQuery({ + projectId: projectId + }); // Query options list: https://cloud.google.com/bigquery/docs/reference/v2/jobs/query const options = { @@ -54,20 +60,28 @@ function queryShakespeare () { }; // Runs the query - return bigquery.query(options) + bigquery + .query(options) .then((results) => { const rows = results[0]; printResult(rows); - return rows; }); } // [END bigquery_simple_app_query] // [END bigquery_simple_app_all] -// [START bigquery_sync_query] -function syncQuery (sqlQuery) { +function syncQuery (sqlQuery, projectId) { + // [START bigquery_sync_query] + // Imports the Google Cloud client library + const BigQuery = require('@google-cloud/bigquery'); + + // The project ID to use, e.g. "your-project-id" + // const projectId = "your-project-id"; + // Instantiates a client - const bigquery = BigQuery(); + const bigquery = BigQuery({ + projectId: projectId + }); // Query options list: https://cloud.google.com/bigquery/docs/reference/v2/jobs/query const options = { @@ -77,20 +91,28 @@ function syncQuery (sqlQuery) { }; // Runs the query - return bigquery.query(options) + bigquery + .query(options) .then((results) => { const rows = results[0]; console.log('Rows:'); rows.forEach((row) => console.log(row)); - return rows; }); + // [END bigquery_sync_query] } -// [END bigquery_sync_query] -// [START bigquery_async_query] -function asyncQuery (sqlQuery) { +function asyncQuery (sqlQuery, projectId) { + // [START bigquery_async_query] + // Imports the Google Cloud client library + const BigQuery = require('@google-cloud/bigquery'); + + // The project ID to use, e.g. "your-project-id" + // const projectId = "your-project-id"; + // Instantiates a client - const bigquery = BigQuery(); + const bigquery = BigQuery({ + projectId: projectId + }); // Query options list: https://cloud.google.com/bigquery/docs/reference/v2/jobs/query const options = { @@ -101,7 +123,8 @@ function asyncQuery (sqlQuery) { let job; // Runs the query as a job - return bigquery.startQuery(options) + bigquery + .startQuery(options) .then((results) => { job = results[0]; console.log(`Job ${job.id} started.`); @@ -115,43 +138,38 @@ function asyncQuery (sqlQuery) { const rows = results[0]; console.log('Rows:'); rows.forEach((row) => console.log(row)); - return rows; }); + // [END bigquery_async_query] } -// [END bigquery_async_query] - -// The command-line program -const cli = require(`yargs`); - -const program = module.exports = { - queryShakespeare: queryShakespeare, - asyncQuery: asyncQuery, - syncQuery: syncQuery, - main: (args) => { - // Run the command-line program - cli.help().strict().parse(args).argv; // eslint-disable-line - } -}; - -cli + +const cli = require(`yargs`) .demand(1) + .options({ + projectId: { + alias: 'p', + default: process.env.GCLOUD_PROJECT || process.env.GOOGLE_CLOUD_PROJECT, + description: 'The Project ID to use. Defaults to the value of the GCLOUD_PROJECT or GOOGLE_CLOUD_PROJECT environment variables.', + requiresArg: true, + type: 'string' + } + }) .command( `sync `, `Run the specified synchronous query.`, {}, - (opts) => program.syncQuery(opts.sqlQuery) + (opts) => syncQuery(opts.sqlQuery, opts.projectId) ) .command( `async `, `Start the specified asynchronous query.`, {}, - (opts) => program.asyncQuery(opts.sqlQuery) + (opts) => asyncQuery(opts.sqlQuery, opts.projectId) ) .command( `shakespeare`, `Queries a public Shakespeare dataset.`, {}, - program.queryShakespeare + (opts) => queryShakespeare(opts.projectId) ) .example( `node $0 sync "SELECT * FROM publicdata.samples.natality LIMIT 5;"`, @@ -167,8 +185,10 @@ cli ) .wrap(120) .recommendCommands() - .epilogue(`For more information, see https://cloud.google.com/bigquery/docs`); + .epilogue(`For more information, see https://cloud.google.com/bigquery/docs`) + .help() + .strict(); if (module === require.main) { - program.main(process.argv.slice(2)); + cli.parse(process.argv.slice(2)); } diff --git a/bigquery/system-test/datasets.test.js b/bigquery/system-test/datasets.test.js index 977c0579b3..e346245268 100644 --- a/bigquery/system-test/datasets.test.js +++ b/bigquery/system-test/datasets.test.js @@ -25,16 +25,15 @@ const cwd = path.join(__dirname, `..`); const cmd = `node datasets.js`; const datasetId = (`nodejs-docs-samples-test-${uuid.v4()}`).replace(/-/gi, '_'); +test.before(tools.checkCredentials); +test.beforeEach(tools.stubConsole); +test.afterEach.always(tools.restoreConsole); test.after.always(async () => { try { await bigquery.dataset(datasetId).delete({ force: true }); } catch (err) {} // ignore error }); -test.before(tools.checkCredentials); -test.beforeEach(tools.stubConsole); -test.afterEach.always(tools.restoreConsole); - test.serial(`should create a dataset`, async (t) => { const output = await tools.runAsync(`${cmd} create ${datasetId}`, cwd); t.is(output, `Dataset ${datasetId} created.`); @@ -43,21 +42,14 @@ test.serial(`should create a dataset`, async (t) => { }); test.serial(`should list datasets`, async (t) => { - await tools.tryTest(async () => { + t.plan(0); + await tools.tryTest(async (assert) => { const output = await tools.runAsync(`${cmd} list`, cwd); - t.true(output.includes(`Datasets:`)); - t.true(output.includes(datasetId)); + assert(output.includes(`Datasets:`)); + assert(output.includes(datasetId)); }).start(); }); -test.serial(`should return the size of a dataset`, async (t) => { - let output = await tools.runAsync(`${cmd} size hacker_news bigquery-public-data`, cwd); - t.true(output.includes(`Size of hacker_news`)); - t.true(output.includes(`MB`)); - output = await tools.runAsync(`${cmd} size ${datasetId}`, cwd); - t.true(output.includes(`Size of ${datasetId}: 0 MB`)); -}); - test.serial(`should delete a dataset`, async (t) => { const output = await tools.runAsync(`${cmd} delete ${datasetId}`, cwd); t.is(output, `Dataset ${datasetId} deleted.`); diff --git a/bigquery/system-test/tables.test.js b/bigquery/system-test/tables.test.js index 433b178994..89c682b8f9 100644 --- a/bigquery/system-test/tables.test.js +++ b/bigquery/system-test/tables.test.js @@ -51,7 +51,8 @@ test.before(async () => { bigquery.createDataset(destDatasetId) ]); }); - +test.beforeEach(tools.stubConsole); +test.afterEach.always(tools.restoreConsole); test.after.always(async () => { try { await bigquery.dataset(srcDatasetId).delete({ force: true }); @@ -74,9 +75,6 @@ test.after.always(async () => { } catch (err) {} // ignore error }); -test.beforeEach(tools.stubConsole); -test.afterEach.always(tools.restoreConsole); - test.serial(`should create a table`, async (t) => { const output = await tools.runAsync(`${cmd} create ${datasetId} ${tableId} "${schema}"`, cwd); t.is(output, `Table ${tableId} created.`); @@ -147,7 +145,7 @@ test.serial(`should insert rows`, async (t) => { const err = await t.throws(tools.runAsync(`${cmd} insert ${datasetId} ${tableId} 'foo.bar'`, cwd)); t.true(err.message.includes(`"json_or_file" (or the file it points to) is not a valid JSON array.`)); const output = await tools.runAsync(`${cmd} insert ${datasetId} ${tableId} '${JSON.stringify(rows)}'`, cwd); - t.is(output, `Inserted:\n{ Name: 'foo', Age: 27, Weight: 80.3, IsMagic: true }\n{ Name: 'bar', Age: 13, Weight: 54.6, IsMagic: false }`); + t.is(output.includes(`Inserted:\n{ Name: 'foo', Age: 27, Weight: 80.3, IsMagic: true }\n{ Name: 'bar', Age: 13, Weight: 54.6, IsMagic: false }`), true); await tools.tryTest(async (assert) => { const [rows] = await bigquery.dataset(datasetId).table(tableId).getRows(); assert.equal(rows.length, 4); diff --git a/bigquery/tables.js b/bigquery/tables.js index 9defd6fc27..9bd0180e51 100644 --- a/bigquery/tables.js +++ b/bigquery/tables.js @@ -15,147 +15,218 @@ 'use strict'; -const BigQuery = require('@google-cloud/bigquery'); -const Storage = require('@google-cloud/storage'); - -// [START bigquery_create_table] function createTable (datasetId, tableId, schema, projectId) { + // [START bigquery_create_table] + // Imports the Google Cloud client library + const BigQuery = require('@google-cloud/bigquery'); + + // The project ID to use, e.g. "your-project-id" + // const projectId = "your-project-id"; + + // The ID of the dataset in which to create the table, e.g. "my_dataset" + // const datasetId = "my_new_dataset"; + + // The ID for the new table, e.g. "my_new_table" + // const tableId = "my_new_table"; + + // The schema of the new table, e.g. "Name:string, Age:integer, Weight:float, IsMagic:boolean" + // const schema = "Name:string, Age:integer, Weight:float, IsMagic:boolean"; + // Instantiates a client const bigquery = BigQuery({ projectId: projectId }); - // References an existing dataset, e.g. "my_dataset" - const dataset = bigquery.dataset(datasetId); - - // Specify a schema, e.g. "Name:string, Age:integer, Weight:float, IsMagic:boolean" // For all options, see https://cloud.google.com/bigquery/docs/reference/v2/tables#resource const options = { schema: schema }; // Create a new table in the dataset - return dataset.createTable(tableId, options) + bigquery + .dataset(datasetId) + .createTable(tableId, options) .then((results) => { const table = results[0]; console.log(`Table ${table.id} created.`); - return table; + }) + .catch((err) => { + console.error('ERROR:', err); }); + // [END bigquery_create_table] } -// [END bigquery_create_table] -// [START bigquery_delete_table] function deleteTable (datasetId, tableId, projectId) { + // [START bigquery_delete_table] + // Imports the Google Cloud client library + const BigQuery = require('@google-cloud/bigquery'); + + // The project ID to use, e.g. "your-project-id" + // const projectId = "your-project-id"; + + // The ID of the dataset of the table to delete, e.g. "my_dataset" + // const datasetId = "my_dataset"; + + // The ID of the table to delete, e.g. "my_table" + // const tableId = "my_table"; + // Instantiates a client const bigquery = BigQuery({ projectId: projectId }); - // References an existing dataset, e.g. "my_dataset" - const dataset = bigquery.dataset(datasetId); - // References an existing table, e.g. "my_table" - const table = dataset.table(tableId); - // Deletes the table - return table.delete() + bigquery + .dataset(datasetId) + .table(tableId) + .delete() .then(() => { - console.log(`Table ${table.id} deleted.`); + console.log(`Table ${tableId} deleted.`); + }) + .catch((err) => { + console.error('ERROR:', err); }); + // [END bigquery_delete_table] } -// [END bigquery_delete_table] -// [START bigquery_list_tables] function listTables (datasetId, projectId) { + // [START bigquery_list_tables] + // Imports the Google Cloud client library + const BigQuery = require('@google-cloud/bigquery'); + + // The project ID to use, e.g. "your-project-id" + // const projectId = "your-project-id"; + // Instantiates a client const bigquery = BigQuery({ projectId: projectId }); - // References an existing dataset, e.g. "my_dataset" - const dataset = bigquery.dataset(datasetId); - // Lists all tables in the dataset - return dataset.getTables() + bigquery + .dataset(datasetId) + .getTables() .then((results) => { const tables = results[0]; console.log('Tables:'); tables.forEach((table) => console.log(table.id)); - return tables; + }) + .catch((err) => { + console.error('ERROR:', err); }); + // [END bigquery_list_tables] } -// [END bigquery_list_tables] -// [START bigquery_browse_table] function browseRows (datasetId, tableId, projectId) { + // [START bigquery_browse_table] + // Imports the Google Cloud client library + const BigQuery = require('@google-cloud/bigquery'); + + // The project ID to use, e.g. "your-project-id" + // const projectId = "your-project-id"; + + // The ID of the dataset of the table to browse, e.g. "my_dataset" + // const datasetId = "my_dataset"; + + // The ID of the table to browse, e.g. "my_table" + // const tableId = "my_table"; + // Instantiates a client const bigquery = BigQuery({ projectId: projectId }); - // References an existing dataset, e.g. "my_dataset" - const dataset = bigquery.dataset(datasetId); - // References an existing table, e.g. "my_table" - const table = dataset.table(tableId); - // Lists rows in the table - return table.getRows() + bigquery + .dataset(datasetId) + .table(tableId) + .getRows() .then((results) => { const rows = results[0]; console.log('Rows:'); rows.forEach((row) => console.log(row)); - return rows; + }) + .catch((err) => { + console.error('ERROR:', err); }); + // [END bigquery_browse_table] } -// [END bigquery_browse_table] -// [START bigquery_copy_table] function copyTable (srcDatasetId, srcTableId, destDatasetId, destTableId, projectId) { + // [START bigquery_copy_table] + // Imports the Google Cloud client library + const BigQuery = require('@google-cloud/bigquery'); + + // The project ID to use, e.g. "your-project-id" + // const projectId = "your-project-id"; + + // The ID of the dataset of the table to copy, e.g. "my_src_dataset" + // const srcDatasetId = "my_src_dataset"; + + // The ID of the table to copy, e.g. "my_src_table" + // const srcTableId = "my_src_table"; + + // The ID of the destination dataset, e.g. "my_dest_dataset" + // const destDatasetId = "my_dest_dataset"; + + // The ID of the destination table, e.g. "my_dest_table" + // const destTableId = "my_dest_table"; + // Instantiates a client const bigquery = BigQuery({ projectId: projectId }); - // References the source dataset, e.g. "my_dataset" - const srcDataset = bigquery.dataset(srcDatasetId); - // References the source table, e.g. "my_table" - const srcTable = srcDataset.table(srcTableId); - // References the destination dataset, e.g. "my_other_dataset" - const destDataset = bigquery.dataset(destDatasetId); - // References the destination table, e.g. "my_other_table" - const destTable = destDataset.table(destTableId); - let job; // Copies the table contents into another table - return srcTable.copy(destTable) + bigquery + .dataset(srcDatasetId) + .table(srcTableId) + .copy(bigquery.dataset(destDatasetId).table(destTableId)) .then((results) => { job = results[0]; console.log(`Job ${job.id} started.`); return job.promise(); }) - .then((results) => { + .then(() => { console.log(`Job ${job.id} completed.`); - return results; + }) + .catch((err) => { + console.error('ERROR:', err); }); + // [END bigquery_copy_table] } -// [END bigquery_copy_table] -// [START bigquery_import_from_file] -function importLocalFile (datasetId, tableId, fileName, projectId) { +function importLocalFile (datasetId, tableId, filename, projectId) { + // [START bigquery_import_from_file] + // Imports the Google Cloud client library + const BigQuery = require('@google-cloud/bigquery'); + + // The project ID to use, e.g. "your-project-id" + // const projectId = "your-project-id"; + + // The name of the file from which data should be imported, e.g. "/path/to/file.csv" + // const filename = "/path/to/file.csv"; + + // The ID of the dataset of the table into which data should be imported, e.g. "my_dataset" + // const datasetId = "my_dataset"; + + // The ID of the table into which data should be imported, e.g. "my_table" + // const tableId = "my_table"; + // Instantiates a client const bigquery = BigQuery({ projectId: projectId }); - // References an existing dataset, e.g. "my_dataset" - const dataset = bigquery.dataset(datasetId); - // References an existing table, e.g. "my_table" - const table = dataset.table(tableId); - let job; // Imports data from a local file into the table - return table.import(fileName) + bigquery + .dataset(datasetId) + .table(tableId) + .import(filename) .then((results) => { job = results[0]; console.log(`Job ${job.id} started.`); @@ -163,34 +234,50 @@ function importLocalFile (datasetId, tableId, fileName, projectId) { }) .then((results) => { console.log(`Job ${job.id} completed.`); - return results; + }) + .catch((err) => { + console.error('ERROR:', err); }); + // [END bigquery_import_from_file] } -// [END bigquery_import_from_file] -// [START bigquery_import_from_gcs] -function importFileFromGCS (datasetId, tableId, bucketName, fileName, projectId) { +function importFileFromGCS (datasetId, tableId, bucketName, filename, projectId) { + // [START bigquery_import_from_gcs] + // Imports the Google Cloud client libraries + const BigQuery = require('@google-cloud/bigquery'); + const Storage = require('@google-cloud/storage'); + + // The project ID to use, e.g. "your-project-id" + // const projectId = "your-project-id"; + + // The ID of the dataset of the table into which data should be imported, e.g. "my_dataset" + // const datasetId = "my_dataset"; + + // The ID of the table into which data should be imported, e.g. "my_table" + // const tableId = "my_table"; + + // The name of the Google Cloud Storage bucket where the file is located, e.g. "my-bucket" + // const bucketName = "my-bucket"; + + // The name of the file from which data should be imported, e.g. "file.csv" + // const filename = "file.csv"; + // Instantiates clients const bigquery = BigQuery({ projectId: projectId }); + const storage = Storage({ projectId: projectId }); - // References an existing dataset, e.g. "my_dataset" - const dataset = bigquery.dataset(datasetId); - // References an existing table, e.g. "my_table" - const table = dataset.table(tableId); - // References an existing bucket, e.g. "my-bucket" - const bucket = storage.bucket(bucketName); - // References an existing file, e.g. "file.txt" - const file = bucket.file(fileName); - let job; - // Imports data from a GCS file into a table - return table.import(file) + // Imports data from a Google Cloud Storage file into the table + bigquery + .dataset(datasetId) + .table(tableId) + .import(storage.bucket(bucketName).file(filename)) .then((results) => { job = results[0]; console.log(`Job ${job.id} started.`); @@ -198,34 +285,50 @@ function importFileFromGCS (datasetId, tableId, bucketName, fileName, projectId) }) .then((results) => { console.log(`Job ${job.id} completed.`); - return results; + }) + .catch((err) => { + console.error('ERROR:', err); }); + // [END bigquery_import_from_gcs] } -// [END bigquery_import_from_gcs] -// [START bigquery_export_gcs] -function exportTableToGCS (datasetId, tableId, bucketName, fileName, projectId) { +function exportTableToGCS (datasetId, tableId, bucketName, filename, projectId) { + // [START bigquery_export_gcs] + // Imports the Google Cloud client libraries + const BigQuery = require('@google-cloud/bigquery'); + const Storage = require('@google-cloud/storage'); + + // The project ID to use, e.g. "your-project-id" + // const projectId = "your-project-id"; + + // The ID of the dataset of the table into which data should be imported, e.g. "my_dataset" + // const datasetId = "my_dataset"; + + // The ID of the table into which data should be imported, e.g. "my_table" + // const tableId = "my_table"; + + // The name of the Google Cloud Storage bucket where the file is located, e.g. "my-bucket" + // const bucketName = "my-bucket"; + + // The name of the file from which data should be imported, e.g. "file.csv" + // const filename = "file.csv"; + // Instantiates clients const bigquery = BigQuery({ projectId: projectId }); + const storage = Storage({ projectId: projectId }); - // References an existing dataset, e.g. "my_dataset" - const dataset = bigquery.dataset(datasetId); - // References an existing table, e.g. "my_table" - const table = dataset.table(tableId); - // References an existing bucket, e.g. "my-bucket" - const bucket = storage.bucket(bucketName); - // References an existing file, e.g. "file.txt" - const file = bucket.file(fileName); - let job; - // Exports data in a table into a Google Cloud Storage file - return table.export(file) + // Exports data from the table into a Google Cloud Storage file + bigquery + .dataset(datasetId) + .table(tableId) + .export(storage.bucket(bucketName).file(filename)) .then((results) => { job = results[0]; console.log(`Job ${job.id} started.`); @@ -233,90 +336,100 @@ function exportTableToGCS (datasetId, tableId, bucketName, fileName, projectId) }) .then((results) => { console.log(`Job ${job.id} completed.`); - return results; + }) + .catch((err) => { + console.error('ERROR:', err); }); + // [END bigquery_export_gcs] } -// [END bigquery_export_gcs] -// [START bigquery_insert_stream] function insertRowsAsStream (datasetId, tableId, rows, projectId) { + // [START bigquery_insert_stream] + // Imports the Google Cloud client library + const BigQuery = require('@google-cloud/bigquery'); + + // The project ID to use, e.g. "your-project-id" + // const projectId = "your-project-id"; + + // The ID of the dataset of the table into which data should be inserted, e.g. "my_dataset" + // const datasetId = "my_dataset"; + + // The ID of the table into which data should be inserted, e.g. "my_table" + // const tableId = "my_table"; + // Instantiates a client const bigquery = BigQuery({ projectId: projectId }); - // References an existing dataset, e.g. "my_dataset" - const dataset = bigquery.dataset(datasetId); - // References an existing table, e.g. "my_table" - const table = dataset.table(tableId); - // Inserts data into a table - return table.insert(rows) + bigquery + .dataset(datasetId) + .table(tableId) + .insert(rows) .then((insertErrors) => { console.log('Inserted:'); rows.forEach((row) => console.log(row)); - return insertErrors; + + if (insertErrors && insertErrors.length > 0) { + console.log('Insert errors:'); + insertErrors.forEach((err) => console.error(err)); + } + }) + .catch((err) => { + console.error('ERROR:', err); }); + // [END bigquery_insert_stream] } -// [END bigquery_insert_stream] -// The command-line program -const cli = require(`yargs`); const fs = require(`fs`); -const program = module.exports = { - createTable: createTable, - listTables: listTables, - browseRows: browseRows, - deleteTable: deleteTable, - importLocalFile: importLocalFile, - importFileFromGCS: importFileFromGCS, - exportTableToGCS: exportTableToGCS, - insertRowsAsStream: insertRowsAsStream, - copyTable: copyTable, - main: (args) => { - // Run the command-line program - cli.help().strict().parse(args).argv; // eslint-disable-line - } -}; - -cli +const cli = require(`yargs`) .demand(1) - .command(`create [projectId]`, `Creates a new table.`, {}, (opts) => { - program.createTable(opts.datasetId, opts.tableId, opts.schema, opts.projectId || process.env.GCLOUD_PROJECT); + .options({ + projectId: { + alias: 'p', + default: process.env.GCLOUD_PROJECT || process.env.GOOGLE_CLOUD_PROJECT, + description: 'The Project ID to use. Defaults to the value of the GCLOUD_PROJECT or GOOGLE_CLOUD_PROJECT environment variables.', + requiresArg: true, + type: 'string' + } + }) + .command(`create `, `Creates a new table.`, {}, (opts) => { + createTable(opts.datasetId, opts.tableId, opts.schema, opts.projectId); }) - .command(`list [projectId]`, `Lists all tables in a dataset.`, {}, (opts) => { - program.listTables(opts.datasetId, opts.projectId || process.env.GCLOUD_PROJECT); + .command(`list `, `Lists all tables in a dataset.`, {}, (opts) => { + listTables(opts.datasetId, opts.projectId); }) - .command(`delete [projectId]`, `Deletes a table.`, {}, (opts) => { - program.deleteTable(opts.datasetId, opts.tableId, opts.projectId || process.env.GCLOUD_PROJECT); + .command(`delete `, `Deletes a table.`, {}, (opts) => { + deleteTable(opts.datasetId, opts.tableId, opts.projectId); }) - .command(`copy [projectId]`, `Makes a copy of a table.`, {}, (opts) => { - program.copyTable(opts.srcDatasetId, opts.srcTableId, opts.destDatasetId, opts.destTableId, opts.projectId || process.env.GCLOUD_PROJECT); + .command(`copy `, `Makes a copy of a table.`, {}, (opts) => { + copyTable(opts.srcDatasetId, opts.srcTableId, opts.destDatasetId, opts.destTableId, opts.projectId); }) - .command(`browse [projectId]`, `Lists rows in a table.`, {}, (opts) => { - program.browseRows(opts.datasetId, opts.tableId, opts.projectId || process.env.GCLOUD_PROJECT); + .command(`browse `, `Lists rows in a table.`, {}, (opts) => { + browseRows(opts.datasetId, opts.tableId, opts.projectId); }) - .command(`import [projectId]`, `Imports data from a local file into a table.`, {}, (opts) => { - program.importLocalFile(opts.datasetId, opts.tableId, opts.fileName, opts.projectId || process.env.GCLOUD_PROJECT); + .command(`import `, `Imports data from a local file into a table.`, {}, (opts) => { + importLocalFile(opts.datasetId, opts.tableId, opts.fileName, opts.projectId); }) - .command(`import-gcs [projectId]`, `Imports data from a Google Cloud Storage file into a table.`, {}, (opts) => { - program.importFileFromGCS(opts.datasetId, opts.tableId, opts.bucketName, opts.fileName, opts.projectId || process.env.GCLOUD_PROJECT); + .command(`import-gcs `, `Imports data from a Google Cloud Storage file into a table.`, {}, (opts) => { + importFileFromGCS(opts.datasetId, opts.tableId, opts.bucketName, opts.fileName, opts.projectId); }) - .command(`export [projectId]`, `Export a table from BigQuery to Google Cloud Storage.`, {}, (opts) => { - program.exportTableToGCS(opts.datasetId, opts.tableId, opts.bucketName, opts.fileName, opts.projectId || process.env.GCLOUD_PROJECT); + .command(`export `, `Export a table from BigQuery to Google Cloud Storage.`, {}, (opts) => { + exportTableToGCS(opts.datasetId, opts.tableId, opts.bucketName, opts.fileName, opts.projectId); }) - .command(`insert [projectId]`, + .command(`insert `, `Insert a JSON array (as a string or newline-delimited file) into a BigQuery table.`, {}, (opts) => { - var content; + let content; try { content = fs.readFileSync(opts.json_or_file); } catch (err) { content = opts.json_or_file; } - var rows = null; + let rows = null; try { rows = JSON.parse(content); } catch (err) {} @@ -325,7 +438,7 @@ cli throw new Error(`"json_or_file" (or the file it points to) is not a valid JSON array.`); } - program.insertRowsAsStream(opts.datasetId, opts.tableId, rows, opts.projectId || process.env.GCLOUD_PROJECT); + insertRowsAsStream(opts.datasetId, opts.tableId, rows, opts.projectId || process.env.GCLOUD_PROJECT); } ) .example( @@ -374,8 +487,10 @@ cli ) .wrap(120) .recommendCommands() - .epilogue(`For more information, see https://cloud.google.com/bigquery/docs`); + .epilogue(`For more information, see https://cloud.google.com/bigquery/docs`) + .help() + .strict(); if (module === require.main) { - program.main(process.argv.slice(2)); + cli.parse(process.argv.slice(2)); } diff --git a/bigquery/yarn.lock b/bigquery/yarn.lock index 5f25e8d74c..a0607c22de 100644 --- a/bigquery/yarn.lock +++ b/bigquery/yarn.lock @@ -72,9 +72,9 @@ string-format-obj "^1.1.0" through2 "^2.0.3" -"@google-cloud/nodejs-repo-tools@1.3.0": - version "1.3.0" - resolved "https://registry.yarnpkg.com/@google-cloud/nodejs-repo-tools/-/nodejs-repo-tools-1.3.0.tgz#2872c3fcc0f0b0d840e1a71ba8f9009761b0df11" +"@google-cloud/nodejs-repo-tools@1.3.2": + version "1.3.2" + resolved "https://registry.yarnpkg.com/@google-cloud/nodejs-repo-tools/-/nodejs-repo-tools-1.3.2.tgz#bb95aa75f15961b8d485fc04f1b3bd5f78ddc8a6" dependencies: async "2.3.0" ava "0.19.1"