diff --git a/packages/bigquery/benchmark/README.md b/packages/bigquery/benchmark/README.md new file mode 100644 index 00000000000..84ebf99d276 --- /dev/null +++ b/packages/bigquery/benchmark/README.md @@ -0,0 +1,8 @@ +# BigQuery Benchmark +This directory contains benchmarks for BigQuery client. + +## Usage +`node bench.js queries.json` + +BigQuery service caches requests so the benchmark should be run +at least twice, disregarding the first result. diff --git a/packages/bigquery/benchmark/bench.js b/packages/bigquery/benchmark/bench.js new file mode 100644 index 00000000000..b982586d314 --- /dev/null +++ b/packages/bigquery/benchmark/bench.js @@ -0,0 +1,83 @@ +/*! + * Copyright 2017 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const async = require('async'); +const fs = require('fs'); +const BigQuery = require('../src/index.js'); +const env = require('../../../system-test/env.js'); + +if (process.argv.length < 3) { + throw new Error(`need query file; ` + + `usage: '${process.argv[0]} ${process.argv[1]} '`); +} + +var queryJson = fs.readFileSync(process.argv[2]); +var queries = JSON.parse(queryJson); +var client = new BigQuery(env); + +var doQuery = function(queryTxt, callback) { + var startMilli = new Date().getTime(); + var numRows = 0; + var numCols; + var timeFirstByteMilli; + + var query = { + query: queryTxt, + useLegacySql: false + }; + + client + .createQueryStream(query) + .on('error', callback) + .on('data', function(row) { + if (numRows === 0) { + numCols = Object.keys(row).length; + timeFirstByteMilli = new Date().getTime() - startMilli; + } else if (numCols !== Object.keys(row).length) { + this.end(); + + var receivedCols = Object.keys(row).length; + var error = new Error( + `query "${queryTxt}": ` + + `wrong number of columns, want ${numCols} got ${receivedCols}` + ); + + callback(error); + } + numRows++; + }) + .on('end', function() { + var timeTotalMilli = new Date().getTime() - startMilli; + + console.log( + `query ${queryTxt}:`, + `got ${numRows} rows,`, + `${numCols} cols,`, + `first byte ${timeFirstByteMilli / 1000} sec,`, + `total ${timeTotalMilli / 1000} sec` + ); + + callback(null); + }); +}; + +async.eachSeries(queries, doQuery, err => { + if (err) { + console.error(err); + } +}); diff --git a/packages/bigquery/benchmark/queries.json b/packages/bigquery/benchmark/queries.json new file mode 100644 index 00000000000..13fed38b52b --- /dev/null +++ b/packages/bigquery/benchmark/queries.json @@ -0,0 +1,10 @@ +[ + "SELECT * FROM `nyc-tlc.yellow.trips` LIMIT 10000", + "SELECT * FROM `nyc-tlc.yellow.trips` LIMIT 100000", + "SELECT * FROM `nyc-tlc.yellow.trips` LIMIT 1000000", + "SELECT title FROM `bigquery-public-data.samples.wikipedia` ORDER BY title LIMIT 1000", + "SELECT title, id, timestamp, contributor_ip FROM `bigquery-public-data.samples.wikipedia` WHERE title like 'Blo%' ORDER BY id", + "SELECT * FROM `bigquery-public-data.baseball.games_post_wide` ORDER BY gameId", + "SELECT * FROM `bigquery-public-data.samples.github_nested` WHERE repository.has_downloads ORDER BY repository.created_at LIMIT 10000", + "SELECT repo_name, path FROM `bigquery-public-data.github_repos.files` WHERE path LIKE '%.java' ORDER BY id LIMIT 1000000" +] diff --git a/packages/bigquery/package.json b/packages/bigquery/package.json index 8415deb55ab..7c42a7a3b80 100644 --- a/packages/bigquery/package.json +++ b/packages/bigquery/package.json @@ -71,7 +71,8 @@ "scripts": { "publish-module": "node ../../scripts/publish.js bigquery", "test": "mocha test/*.js", - "system-test": "mocha system-test/*.js --no-timeouts --bail" + "system-test": "mocha system-test/*.js --no-timeouts --bail", + "benchmark": "time node benchmark/bench.js benchmark/queries.json" }, "license": "Apache-2.0", "engines": {