From 96eb260dd4f87b5610e83a0c74a70bee0a53026f Mon Sep 17 00:00:00 2001 From: Dave Gramlich Date: Mon, 11 Jul 2016 17:23:52 -0400 Subject: [PATCH] bigtable: initial support (#1279) * bigtable: initial support add support for scan filters add examples via jsdoc add bigtable to docs site * docs: updated Bigtable README example --- CONTRIBUTING.md | 2 + README.md | 57 ++ docs/json/master/bigtable/.gitkeep | 0 docs/toc.json | 16 + lib/bigtable/family.js | 312 ++++++++++ lib/bigtable/filter.js | 889 +++++++++++++++++++++++++++ lib/bigtable/index.js | 475 +++++++++++++++ lib/bigtable/mutation.js | 332 ++++++++++ lib/bigtable/row.js | 750 +++++++++++++++++++++++ lib/bigtable/table.js | 857 ++++++++++++++++++++++++++ lib/common/grpc-service-object.js | 11 +- lib/common/grpc-service.js | 236 ++++++-- lib/index.js | 20 + lib/pubsub/subscription.js | 2 +- package.json | 5 +- scripts/docs.js | 3 +- system-test/bigtable.js | 710 ++++++++++++++++++++++ test/bigtable/family.js | 329 ++++++++++ test/bigtable/filter.js | 624 +++++++++++++++++++ test/bigtable/index.js | 325 ++++++++++ test/bigtable/mutation.js | 412 +++++++++++++ test/bigtable/row.js | 801 ++++++++++++++++++++++++ test/bigtable/table.js | 941 +++++++++++++++++++++++++++++ test/common/grpc-service-object.js | 14 + test/common/grpc-service.js | 418 ++++++++++++- test/index.js | 15 + 26 files changed, 8512 insertions(+), 44 deletions(-) create mode 100644 docs/json/master/bigtable/.gitkeep create mode 100644 lib/bigtable/family.js create mode 100644 lib/bigtable/filter.js create mode 100644 lib/bigtable/index.js create mode 100644 lib/bigtable/mutation.js create mode 100644 lib/bigtable/row.js create mode 100644 lib/bigtable/table.js create mode 100644 system-test/bigtable.js create mode 100644 test/bigtable/family.js create mode 100644 test/bigtable/filter.js create mode 100644 test/bigtable/index.js create mode 100644 test/bigtable/mutation.js create mode 100644 test/bigtable/row.js create mode 100644 test/bigtable/table.js diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 28456142eee..2aa55aef3f7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,6 +23,8 @@ To run the system tests, first create and configure a project in the Google Deve - **GCLOUD_TESTS_KEY**: The path to the JSON key file. - ***GCLOUD_TESTS_API_KEY*** (*optional*): An API key that can be used to test the Translate API. - ***GCLOUD_TESTS_DNS_DOMAIN*** (*optional*): A domain you own managed by Google Cloud DNS (expected format: `'gcloud-node.com.'`). +- ***GCLOUD_TESTS_BIGTABLE_ZONE*** (*optional*): A zone containing a Google Cloud Bigtable cluster. +- ***GCLOUD_TESTS_BIGTABLE_CLUSTER*** (*optional*): A cluster used to create Bigtable Tables on. Install the [gcloud command-line tool][gcloudcli] to your machine and use it to create the indexes used in the datastore system tests with indexes found in `system-test/data/index/yaml`: diff --git a/README.md b/README.md index 42e9339e09e..6e58cf9ea26 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ This client supports the following Google Cloud Platform services: * [Google BigQuery](#google-bigquery) +* [Google Cloud Bigtable](#google-cloud-bigtable) * [Google Cloud Datastore](#google-cloud-datastore) * [Google Cloud DNS](#google-cloud-dns) * [Google Cloud Pub/Sub](#google-cloud-pubsub) @@ -120,6 +121,58 @@ job.getQueryResults().on('data', function(row) {}); ``` +## Google Cloud Bigtable + +- [API Documentation][gcloud-bigtable-docs] +- [Official Documentation][cloud-bigtable-docs] + +*You may need to [create a cluster][gcloud-bigtable-cluster] to use the Google Cloud Bigtable API with your project.* + +#### Preview + +```js +var gcloud = require('gcloud'); + +// Authenticating on a per-API-basis. You don't need to do this if you auth on a +// global basis (see Authentication section above). +var bigtable = gcloud.bigtable({ + projectId: 'my-project', + keyFilename: '/path/to/keyfile.json', + zone: 'my-zone', + cluster: 'my-cluster' +}); + +var table = bigtable.table('prezzy'); + +table.getRows(function(err, rows) {}); + +// Update a row in your table. +var row = table.row('alincoln'); + +row.save('follows:gwashington', 1, function(err) { + if (err) { + // Error handling omitted. + } + + row.get('follows:gwashington', function(err, data) { + if (err) { + // Error handling omitted. + } + + // data = { + // follows: { + // gwashington: [ + // { + // value: 1 + // } + // ] + // } + // } + }); +}); +``` + + ## Google Cloud Datastore - [API Documentation][gcloud-datastore-docs] @@ -658,6 +711,7 @@ Apache 2.0 - See [COPYING](COPYING) for more information. [gcloud-homepage]: https://googlecloudplatform.github.io/gcloud-node/ [gcloud-docs]: https://googlecloudplatform.github.io/gcloud-node/#/docs [gcloud-bigquery-docs]: https://googlecloudplatform.github.io/gcloud-node/#/docs/bigquery +[gcloud-bigtable-docs]: https://googlecloudplatform.github.io/gcloud-node/#/docs/bigtable [gcloud-compute-docs]: https://googlecloudplatform.github.io/gcloud-node/#/docs/compute [gcloud-datastore-docs]: https://googlecloudplatform.github.io/gcloud-node/#/docs/datastore [gcloud-dns-docs]: https://googlecloudplatform.github.io/gcloud-node/#/docs/dns @@ -685,6 +739,9 @@ Apache 2.0 - See [COPYING](COPYING) for more information. [cloud-bigquery-docs]: https://cloud.google.com/bigquery/what-is-bigquery +[cloud-bigtable-docs]: https://cloud.google.com/bigtable/docs/ +[cloud-bigtable-cluster]: https://cloud.google.com/bigtable/docs/creating-cluster + [cloud-compute-docs]: https://cloud.google.com/compute/docs [cloud-datastore-docs]: https://cloud.google.com/datastore/docs diff --git a/docs/json/master/bigtable/.gitkeep b/docs/json/master/bigtable/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docs/toc.json b/docs/toc.json index 443f6107b7a..dcbdc7537fe 100644 --- a/docs/toc.json +++ b/docs/toc.json @@ -46,6 +46,22 @@ "title": "Job", "type": "bigquery/job" }] + }, { + "title": "Bigtable", + "type": "bigtable", + "nav": [{ + "title": "Table", + "type": "bigtable/table" + }, { + "title": "Family", + "type": "bigtable/family" + }, { + "title": "Row", + "type": "bigtable/row" + }, { + "title": "Filter", + "type": "bigtable/filter" + }] }, { "title": "Compute", "type": "compute", diff --git a/lib/bigtable/family.js b/lib/bigtable/family.js new file mode 100644 index 00000000000..0ac04d31da3 --- /dev/null +++ b/lib/bigtable/family.js @@ -0,0 +1,312 @@ +/*! + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /*! + * @module bigtable/family + */ + +'use strict'; + +var createErrorClass = require('create-error-class'); +var nodeutil = require('util'); +var is = require('is'); + +/** + * @private + */ +var FamilyError = createErrorClass('FamilyError', function(name) { + this.message = 'Column family not found: ' + name + '.'; + this.code = 404; +}); + +/** + * @type {module:common/grpcServiceObject} + * @private + */ +var GrpcServiceObject = require('../common/grpc-service-object.js'); + +/** + * Create a Family object to interact with your table column families. + * + * @constructor + * @alias module:bigtable/family + * + * @example + * var gcloud = require('gcloud'); + * + * var bigtable = gcloud.bigtable({ + * keyFilename: '/path/to/keyfile.json', + * projectId: 'grape-spaceship-123', + * cluster: 'gcloud-node', + * zone: 'us-central1-b' + * }); + * + * var table = bigtable.table('prezzy'); + * var family = table.family('follows'); + */ +function Family(table, name) { + var id = Family.formatName_(table.id, name); + + var methods = { + + /** + * Create a column family. + * + * @param {object=} options - See {module:bigtable/table#createFamily}. + * + * @example + * family.create(function(err, family, apiResponse) { + * // The column family was created successfully. + * }); + */ + create: true, + + /** + * Delete the column family. + * + * @param {function=} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * family.delete(function(err, apiResponse) {}); + */ + delete: { + protoOpts: { + service: 'BigtableTableService', + method: 'deleteColumnFamily' + }, + reqOpts: { + name: id + } + }, + + /** + * Check if the column family exists. + * + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {boolean} callback.exists - Whether the family exists or not. + * + * @example + * family.exists(function(err, exists) {}); + */ + exists: true, + + /** + * Get a column family if it exists. + * + * You may optionally use this to "get or create" an object by providing an + * object with `autoCreate` set to `true`. Any extra configuration that is + * normally required for the `create` method must be contained within this + * object as well. + * + * @param {options=} options - Configuration object. + * @param {boolean} options.autoCreate - Automatically create the object if + * it does not exist. Default: `false` + * + * @example + * family.get(function(err, family, apiResponse) { + * // `family.metadata` has been populated. + * }); + */ + get: true + }; + + var config = { + parent: table, + id: id, + methods: methods, + createMethod: function(_, options, callback) { + table.createFamily(name, options, callback); + } + }; + + GrpcServiceObject.call(this, config); +} + +nodeutil.inherits(Family, GrpcServiceObject); + +/** + * Format the Column Family name into the expected proto format. + * + * @private + * + * @param {string} tableName - The full formatted table name. + * @param {string} name - The column family name. + * @return {string} + * + * @example + * Family.formatName_( + * 'projects/p/zones/z/clusters/c/tables/t', + * 'my-family' + * ); + * // 'projects/p/zones/z/clusters/c/tables/t/columnFamilies/my-family' + */ +Family.formatName_ = function(tableName, name) { + if (name.indexOf('/') > -1) { + return name; + } + + return tableName + '/columnFamilies/' + name; +}; + +/** + * Formats Garbage Collection rule into proto format. + * + * @private + * + * @param {object} ruleObj - The rule object. + * @return {object} + * + * @example + * Family.formatRule({ + * age: { + * seconds: 10000, + * nanos: 10000 + * }, + * versions: 2, + * union: true + * }); + * // { + * // union: { + * // rules: [ + * // { + * // maxAge: { + * // seconds: 10000, + * // nanos: 10000 + * // } + * // }, { + * // maxNumVersions: 2 + * // } + * // ] + * // } + * // } + */ +Family.formatRule_ = function(ruleObj) { + var rules = []; + + if (ruleObj.age) { + rules.push({ + maxAge: ruleObj.age + }); + } + + if (ruleObj.versions) { + rules.push({ + maxNumVersions: ruleObj.versions + }); + } + + if (!ruleObj.intersection && !ruleObj.union) { + return rules[0]; + } + + var rule = {}; + var ruleType = ruleObj.union ? 'union' : 'intersection'; + + rule[ruleType] = { + rules: rules + }; + + return rule; +}; + +/** + * Get the column family's metadata. + * + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {object} callback.metadata - The metadata. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * family.getMetadata(function(err, metadata, apiResponse) {}); + */ +Family.prototype.getMetadata = function(callback) { + var self = this; + + this.parent.getFamilies(function(err, families, resp) { + if (err) { + callback(err, null, resp); + return; + } + + for (var i = 0, l = families.length; i < l; i++) { + if (families[i].id === self.id) { + self.metadata = families[i].metadata; + callback(null, self.metadata, resp); + return; + } + } + + var error = new FamilyError(self.id); + callback(error, null, resp); + }); +}; + +/** + * Set the column family's metadata. + * + * See {module:bigtable/table#createFamily} for a detailed explanation of the + * arguments. + * + * @resource [Garbage Collection Proto Docs]{@link https://github.com/googleapis/googleapis/blob/3592a7339da5a31a3565870989beb86e9235476e/google/bigtable/admin/table/v1/bigtable_table_data.proto#L59} + * + * @param {object} metadata - Metadata object. + * @param {object|string=} metadata.rule - Garbage collection rule. + * @param {string=} metadata.name - The updated column family name. + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * family.setMetadata({ + * name: 'updated-name', + * rule: 'version() > 3 || (age() > 3d && version() > 1)' + * }, function(err, apiResponse) {}); + */ +Family.prototype.setMetadata = function(metadata, callback) { + var grpcOpts = { + service: 'BigtableTableService', + method: 'updateColumnFamily' + }; + + var reqOpts = { + name: this.id + }; + + if (metadata.rule) { + if (is.string(metadata.rule)) { + reqOpts.gcExpression = metadata.rule; + } else if (is.object(metadata.rule)) { + reqOpts.gcRule = Family.formatRule_(metadata.rule); + } + } + + if (metadata.name) { + reqOpts.name = Family.formatName_(this.parent.id, metadata.name); + } + + this.request(grpcOpts, reqOpts, callback); +}; + +module.exports = Family; +module.exports.FamilyError = FamilyError; diff --git a/lib/bigtable/filter.js b/lib/bigtable/filter.js new file mode 100644 index 00000000000..5a1c860b655 --- /dev/null +++ b/lib/bigtable/filter.js @@ -0,0 +1,889 @@ +/*! + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*! + * @module bigtable/filter + */ + +'use strict'; + +var arrify = require('arrify'); +var is = require('is'); +var createErrorClass = require('create-error-class'); +var extend = require('extend'); + +/** + * @private + * @type {module:bigtable/mutation} + */ +var Mutation = require('./mutation.js'); + +/** + * @private + */ +var FilterError = createErrorClass('FilterError', function(filter) { + this.message = 'Unknown filter: ' + filter + '.'; +}); + +/** + * A filter takes a row as input and produces an alternate view of the row based + * on specified rules. For example, a row filter might trim down a row to + * include just the cells from columns matching a given regular expression, or + * might return all the cells of a row but not their values. More complicated + * filters can be composed out of these components to express requests such as, + * "within every column of a particular family, give just the two most recent + * cells which are older than timestamp X." + * + * There are two broad categories of filters (true filters and transformers), + * as well as two ways to compose simple filters into more complex ones + * ({module:bigtable/filter#interleave}). They work as follows: + * + * True filters alter the input row by excluding some of its cells wholesale + * from the output row. An example of a true filter is the + * {module:bigtable/filter#value} filter, which excludes cells whose values + * don't match the specified pattern. All regex true filters use RE2 syntax + * (https://github.com/google/re2/wiki/Syntax) and are evaluated as full + * matches. An important point to keep in mind is that RE2(.) is equivalent by + * default to RE2([^\n]), meaning that it does not match newlines. When + * attempting to match an arbitrary byte, you should therefore use the escape + * sequence '\C', which may need to be further escaped as '\\C' in your client + * language. + * + * Transformers alter the input row by changing the values of some of its + * cells in the output, without excluding them completely. Currently, the only + * supported transformer is the {module:bigtable/filter#value} `strip` filter, + * which replaces every cell's value with the empty string. + * + * The total serialized size of a filter message must not + * exceed 4096 bytes, and filters may not be nested within each other to a depth + * of more than 20. + * + * Use the following table for the various examples found throughout the + * filter documentation. + * + * | Row Key | follows:gwashington | follows:jadams | follows:tjefferson | + * | ----------- |:-------------------:|:--------------:|:------------------:| + * | gwashington | | 1 | | + * | tjefferson | 1 | 1 | | + * | jadams | 1 | | 1 | + * + * @constructor + * @alias module:bigtable/filter + */ +function Filter() { + this.filters_ = []; +} + +/** + * @private + * @throws TypeError + * + * Transforms Arrays into a simple regular expression for matching multiple + * values. + * + * @param {regex|string|string[]} regex - Either a plain regex, a regex in + * string form or an array of strings. + * + * @return {string} + * + * @example + * var regexString = Filter.convertToRegExpString(['a', 'b', 'c']); + * // => '(a|b|c)' + */ +Filter.convertToRegExpString = function(regex) { + if (is.regexp(regex)) { + return regex.toString().replace(/^\/|\/$/g, ''); + } + + if (is.array(regex)) { + return '(' + regex.join('|') + ')'; + } + + if (is.string(regex)) { + return regex; + } + + if (is.number(regex)) { + return regex.toString(); + } + + throw new TypeError('Can\'t convert to RegExp String from unknown type.'); +}; + +/** + * @private + * + * Creates a range object. All bounds default to inclusive. + * + * @param {?object|string} start - Lower bound value. + * @param {?object|string} end - Upper bound value. + * @param {string} key - Key used to create range value keys. + * + * @return {object} + * + * @example + * var range = Filter.createRange('value1', 'value2', 'Test'); + * // { + * // startTestInclusive: new Buffer('value1'), + * // endTestExclusive: new Buffer('value2') + * // } + * + * //- + * // It's also possible to pass in objects to specify inclusive/exclusive + * // bounds. + * //- + * var upperBound = { + * value: 'value3', + * inclusive: false + * }; + * + * var range = Filter.createRange(upperBound, null, 'Test2'); + * // => { + * // startTest2Exclusive: 'value3' + * // } + */ +Filter.createRange = function(start, end, key) { + var range = {}; + + if (start) { + extend(range, createBound('start', start, key)); + } + + if (end) { + extend(range, createBound('end', end, key)); + } + + return range; + + function createBound(boundName, boundData, key) { + var isInclusive = boundData.inclusive !== false; + var boundKey = boundName + key + (isInclusive ? 'Inclusive' : 'Exclusive'); + var bound = {}; + + bound[boundKey] = Mutation.convertToBytes(boundData.value || boundData); + return bound; + } +}; + +/** + * @private + * @throws FilterError + * + * Turns filters into proto friendly format. + * + * @param {object[]} filters - The list of filters to be parsed. + * + * @return {object} + * + * @example + * var filter = Filter.parse([ + * { + * family: 'my-family', + * }, { + * column: 'my-column' + * } + * ]); + * // { + * // chain: { + * // filters: [ + * // { + * // familyNameRegexFilter: 'my-family' + * // }, + * // { + * // columnQualifierRegexFilter: 'my-column' + * // } + * // ] + * // } + * // } + */ +Filter.parse = function(filters) { + var filter = new Filter(); + + arrify(filters).forEach(function(filterObj) { + var key = Object.keys(filterObj)[0]; + + if (!is.function(filter[key])) { + throw new FilterError(key); + } + + filter[key](filterObj[key]); + }); + + return filter.toProto(); +}; + +/** + * @example + * //- + * // Matches all cells, regardless of input. Functionally equivalent to + * // leaving `filter` unset, but included for completeness. + * //- + * var filter = { + * all: true + * }; + * + * //- + * // Does not match any cells, regardless of input. Useful for temporarily + * // disabling just part of a filter. + * //- + * var filter = { + * all: false + * }; + */ +Filter.prototype.all = function(pass) { + var filterName = pass ? 'passAllFilter' : 'blockAllFilter'; + + this.set(filterName, true); +}; + +/** + * Matches only cells from columns whose qualifiers satisfy the given RE2 + * regex. + * + * Note that, since column qualifiers can contain arbitrary bytes, the '\C' + * escape sequence must be used if a true wildcard is desired. The '.' + * character will not match the new line character '\n', which may be + * present in a binary qualifier. + * + * @example + * //- + * // Using the following filter, we would retrieve the `tjefferson` and + * // `gwashington` columns. + * //- + * var filter = [ + * { + * column: /[a-z]+on$/ + * } + * ]; + * + * //- + * // You can also provide a string (optionally containing regexp characters) + * // for simple column filters. + * //- + * var filter = [ + * { + * column: 'gwashington' + * } + * ]; + * + * //- + * // Or you can provide an array of strings if you wish to match against + * // multiple columns. + * //- + * var filter = [ + * { + * column: [ + * 'gwashington', + * 'tjefferson' + * ] + * } + * ]; + * + * //- + * // If you wish to use additional column filters, consider using the following + * // syntax. + * //- + * var filter = [ + * { + * column: { + * name: 'gwashington' + * } + * } + * ]; + * + * + * //- + * //

Column Cell Limits

+ * // + * // Matches only the most recent number of versions within each column. For + * // example, if the `versions` is set to 2, this filter would only match + * // columns updated at the two most recent timestamps. + * // + * // If duplicate cells are present, as is possible when using an + * // {module:bigtable/filter#interleave} filter, each copy of the cell is + * // counted separately. + * //- + * var filter = [ + * { + * column: { + * cellLimit: 2 + * } + * } + * ]; + * + * //- + * //

Column Ranges

+ * // + * // Specifies a contiguous range of columns within a single column family. + * // The range spans from : to + * // :, where both bounds can be either + * // inclusive or exclusive. By default both are inclusive. + * // + * // When the `start` bound is omitted it is interpreted as an empty string. + * // When the `end` bound is omitted it is interpreted as Infinity. + * //- + * var filter = [ + * { + * column: { + * family: 'follows', + * start: 'gwashington', + * end: 'tjefferson' + * } + * } + * ]; + * + * //- + * // By default, both the `start` and `end` bounds are inclusive. You can + * // override these by providing an object explicity stating whether or not it + * // is `inclusive`. + * //- + * var filter = [ + * { + * column: { + * family: 'follows', + * start: { + * value: 'gwashington', + * inclusive: false + * }, + * end: { + * value: 'jadams', + * inclusive: false + * } + * } + * } + * ]; + */ +Filter.prototype.column = function(column) { + if (!is.object(column)) { + column = { + name: column + }; + } + + if (column.name) { + var name = Filter.convertToRegExpString(column.name); + + name = Mutation.convertToBytes(name); + this.set('columnQualifierRegexFilter', name); + } + + if (is.number(column.cellLimit)) { + this.set('cellsPerColumnLimitFilter', column.cellLimit); + } + + if (column.start || column.end) { + var range = Filter.createRange(column.start, column.end, 'Qualifier'); + + range.familyName = column.family; + this.set('columnRangeFilter', range); + } +}; + +/** + * A filter which evaluates one of two possible filters, depending on + * whether or not a `test` filter outputs any cells from the input row. + * + * IMPORTANT NOTE: The `test` filter does not execute atomically with the + * pass and fail filters, which may lead to inconsistent or unexpected + * results. Additionally, condition filters have poor performance, especially + * when filters are set for the fail condition. + * + * @example + * //- + * // In the following example we're creating a filter that will check if + * // `gwashington` follows `tjefferson`. If he does, we'll get all of the + * // `gwashington` data. If he does not, we'll instead return all of the + * // `tjefferson` data. + * //- + * var filter = [ + * { + * condition: { + * // If `test` outputs any cells, then `pass` will be evaluated on the + * // input row. Otherwise `fail` will be evaluated. + * test: [ + * { + * row: 'gwashington' + * }, + * { + * family: 'follows' + * }, + * { + * column: 'tjefferson' + * } + * ], + * + * // If omitted, no results will be returned in the true case. + * pass: [ + * { + * row: 'gwashington' + * } + * ], + * + * // If omitted, no results will be returned in the false case. + * fail: [ + * { + * row: 'tjefferson' + * } + * ] + * } + * } + * ]; + */ +Filter.prototype.condition = function(condition) { + this.set('condition', { + predicateFilter: Filter.parse(condition.test), + trueFilter: Filter.parse(condition.pass), + falseFilter: Filter.parse(condition.fail) + }); +}; + +/** + * Matches only cells from columns whose families satisfy the given RE2 + * regex. For technical reasons, the regex must not contain the ':' + * character, even if it is not being used as a literal. + * Note that, since column families cannot contain the new line character + * '\n', it is sufficient to use '.' as a full wildcard when matching + * column family names. + * + * @example + * var filter = [ + * { + * family: 'follows' + * } + * ]; + */ +Filter.prototype.family = function(family) { + family = Filter.convertToRegExpString(family); + this.set('familyNameRegexFilter', family); +}; + +/** + * Applies several filters to the data in parallel and combines the results. + * + * The elements of "filters" all process a copy of the input row, and the + * results are pooled, sorted, and combined into a single output row. + * If multiple cells are produced with the same column and timestamp, + * they will all appear in the output row in an unspecified mutual order. + * All interleaved filters are executed atomically. + * + * @example + * //- + * // In the following example, we're creating a filter that will retrieve + * // results for entries that were either created between December 17th, 2015 + * // and March 22nd, 2016 or entries that have data for `follows:tjefferson`. + * //- + * var filter = [ + * { + * interleave: [ + * [ + * { + * time: { + * start: new Date('December 17, 2015'), + * end: new Date('March 22, 2016') + * } + * } + * ], + * [ + * { + * family: 'follows' + * }, + * { + * column: 'tjefferson' + * } + * ] + * ] + * } + * ]; + */ +Filter.prototype.interleave = function(filters) { + this.set('interleave', { + filters: filters.map(Filter.parse) + }); +}; + +/** + * Applies the given label to all cells in the output row. This allows + * the client to determine which results were produced from which part of + * the filter. + * + * Values must be at most 15 characters in length, and match the RE2 + * pattern [a-z0-9\\-]+ + * + * Due to a technical limitation, it is not currently possible to apply + * multiple labels to a cell. As a result, a chain filter may have no more than + * one sub-filter which contains a apply label transformer. It is okay for + * an {module:bigtable/filter#interleave} to contain multiple apply label + * transformers, as they will be applied to separate copies of the input. This + * may be relaxed in the future. + * + * @example + * var filter = { + * label: 'my-label' + * }; + */ +Filter.prototype.label = function(label) { + this.set('applyLabelTransformer', label); +}; + +/** + * @example + * //- + * // Matches only cells from rows whose keys satisfy the given RE2 regex. In + * // other words, passes through the entire row when the key matches, and + * // otherwise produces an empty row. + * // + * // Note that, since row keys can contain arbitrary bytes, the '\C' escape + * // sequence must be used if a true wildcard is desired. The '.' character + * // will not match the new line character '\n', which may be present in a + * // binary key. + * // + * // In the following example we'll use a regular expression to match all + * // row keys ending with the letters "on", which would then yield + * // `gwashington` and `tjefferson`. + * //- + * var filter = [ + * { + * row: /[a-z]+on$/ + * } + * ]; + * + * //- + * // You can also provide a string (optionally containing regexp characters) + * // for simple key filters. + * //- + * var filter = [ + * { + * row: 'gwashington' + * } + * ]; + * + * //- + * // Or you can provide an array of strings if you wish to match against + * // multiple keys. + * //- + * var filter = [ + * { + * row: [ + * 'gwashington', + * 'tjefferson' + * ] + * } + * ]; + * + * //- + * // If you wish to use additional row filters, consider using the following + * // syntax. + * //- + * var filter = [ + * { + * row: { + * key: 'gwashington' + * } + * } + * ]; + * + * //- + * //

Row Samples

+ * // + * // Matches all cells from a row with probability p, and matches no cells + * // from the row with probability 1-p. + * //- + * var filter = [ + * { + * row: { + * sample: 1 + * } + * } + * ]; + * + * //- + * //

Row Cell Offsets

+ * // + * // Skips the first N cells of each row, matching all subsequent cells. + * // If duplicate cells are present, as is possible when using an + * // {module:bigtable/filter#interleave}, each copy of the cell is counted + * // separately. + * //- + * var filter = [ + * { + * row: { + * cellOffset: 2 + * } + * } + * ]; + * + * //- + * //

Row Cell Limits

+ * // + * // Matches only the first N cells of each row. + * // If duplicate cells are present, as is possible when using an + * // {module:bigtable/filter#interleave}, each copy of the cell is counted + * // separately. + * //- + * var filter = [ + * { + * row: { + * cellLimit: 4 + * } + * } + * ]; + */ +Filter.prototype.row = function(row) { + if (!is.object(row)) { + row = { + key: row + }; + } + + if (row.key) { + var key = Filter.convertToRegExpString(row.key); + + key = Mutation.convertToBytes(key); + this.set('rowKeyRegexFilter', key); + } + + if (row.sample) { + this.set('rowSampleFilter', row.sample); + } + + if (is.number(row.cellOffset)) { + this.set('cellsPerRowOffsetFilter', row.cellOffset); + } + + if (is.number(row.cellLimit)) { + this.set('cellsPerRowLimitFilter', row.cellLimit); + } +}; + +/** + * Stores a filter object. + * + * @private + * + * @param {string} key - Filter name. + * @param {*} value - Filter value. + */ +Filter.prototype.set = function(key, value) { + var filter = {}; + + filter[key] = value; + this.filters_.push(filter); +}; + +/** + * This filter is meant for advanced use only. Hook for introspection into the + * filter. Outputs all cells directly to the output of the read rather than to + * any parent filter. + * + * Despite being excluded by the qualifier filter, a copy of every cell that + * reaches the sink is present in the final result. + * + * As with an {module:bigtable/filter#interleave} filter, duplicate cells are + * possible, and appear in an unspecified mutual order. + * + * Cannot be used within {module:bigtable/filter#condition} filter. + * + * @example + * //- + * // Using the following filter, a copy of every cell that reaches the sink is + * // present in the final result, despite being excluded by the qualifier + * // filter + * //- + * var filter = [ + * { + * family: 'follows' + * }, + * { + * interleave: [ + * [ + * { + * all: true + * } + * ], + * [ + * { + * label: 'prezzy' + * }, + * { + * sink: true + * } + * ] + * ] + * }, + * { + * column: 'gwashington' + * } + * ]; + * + * //- + * // As with an {module:bigtable/filter#interleave} filter, duplicate cells + * // are possible, and appear in an unspecified mutual order. In this case we + * // have a duplicates with multiple `gwashington` columns because one copy + * // passed through the {module:bigtable/filter#all} filter while the other was + * // passed through the {module:bigtable/filter#label} and sink. Note that one + * // copy has label "prezzy" while the other does not. + * //- + */ +Filter.prototype.sink = function(sink) { + this.set('sink', sink); +}; + +/** + * Matches only cells with timestamps within the given range. + * + * @example + * var filter = [ + * { + * time: { + * start: new Date('December 17, 2006 03:24:00'), + * end: new Date() + * } + * } + * ]; + */ +Filter.prototype.time = function(time) { + var range = Mutation.createTimeRange(time.start, time.end); + this.set('timestampRangeFilter', range); +}; + +/** + * @private + * If we detect multiple filters, we'll assume it's a chain filter and the + * execution of the filters will be the order in which they were specified. + */ +Filter.prototype.toProto = function() { + if (this.filters_.length === 1) { + return this.filters_[0]; + } + + return { + chain: { + filters: this.filters_ + } + }; +}; + +/** + * Matches only cells with values that satisfy the given regular expression. + * Note that, since cell values can contain arbitrary bytes, the '\C' escape + * sequence must be used if a true wildcard is desired. The '.' character + * will not match the new line character '\n', which may be present in a + * binary value. + * + * @example + * var filter = [ + * { + * value: /[0-9]/ + * } + * ]; + * + * //- + * // You can also provide a string (optionally containing regexp characters) + * // for value filters. + * //- + * var filter = [ + * { + * value: '1' + * } + * ]; + * + * //- + * // Or you can provide an array of strings if you wish to match against + * // multiple values. + * //- + * var filter = [ + * { + * value: ['1', '9'] + * } + * ]; + * + * //- + * //

Value Ranges

+ * // + * // Specifies a contigous range of values. + * // + * // When the `start` bound is omitted it is interpreted as an empty string. + * // When the `end` bound is omitted it is interpreted as Infinity. + * //- + * var filter = [ + * { + * value: { + * start: '1', + * end: '9' + * } + * } + * ]; + * + * //- + * // By default, both the `start` and `end` bounds are inclusive. You can + * // override these by providing an object explicity stating whether or not it + * // is `inclusive`. + * //- + * var filter = [ + * { + * value: { + * start: { + * value: '1', + * inclusive: false + * }, + * end: { + * value: '9', + * inclusive: false + * } + * } + * } + * ]; + * + * //- + * //

Strip Values

+ * // + * // Replaces each cell's value with an emtpy string. + * //- + * var filter = [ + * { + * value: { + * strip: true + * } + * } + * ]; + */ +Filter.prototype.value = function(value) { + if (!is.object(value)) { + value = { + value: value + }; + } + + if (value.value) { + var valueReg = Filter.convertToRegExpString(value.value); + + valueReg = Mutation.convertToBytes(valueReg); + this.set('valueRegexFilter', valueReg); + } + + if (value.start || value.end) { + var range = Filter.createRange(value.start, value.end, 'Value'); + + this.set('valueRangeFilter', range); + } + + if (value.strip) { + this.set('stripValueTransformer', value.strip); + } +}; + +module.exports = Filter; +module.exports.FilterError = FilterError; diff --git a/lib/bigtable/index.js b/lib/bigtable/index.js new file mode 100644 index 00000000000..32a80c5c41e --- /dev/null +++ b/lib/bigtable/index.js @@ -0,0 +1,475 @@ +/*! + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*! + * @module bigtable + */ + +'use strict'; + +var googleProtoFiles = require('google-proto-files'); +var is = require('is'); +var nodeutil = require('util'); +var format = require('string-format-obj'); + +/** + * @type {module:bigtable/table} + * @private + */ +var Table = require('./table.js'); + +/** + * @type {module:common/grpcService} + * @private + */ +var GrpcService = require('../common/grpc-service.js'); + +/** + * @type {module:common/util} + * @private + */ +var util = require('../common/util.js'); + +/** + * Interact with + * [Google Cloud Bigtable](https://cloud.google.com/bigtable/). + * + * @constructor + * @alias module:bigtable + * + * @classdesc + * The `gcloud.bigtable` object allows you interact with Google Cloud Bigtable. + * + * To learn more about Bigtable, read the + * [Google Cloud Bigtable Concepts Overview](https://cloud.google.com/bigtable/docs/concepts) + * + * @resource [Creating a Cloud Bigtable Cluster]{@link https://cloud.google.com/bigtable/docs/creating-cluster} + * + * @param {object=} options - [Configuration object](#/docs). + * @param {string} options.cluster - The cluster name that hosts your tables. + * @param {string|module:compute/zone} options.zone - The zone in which your + * cluster resides. + * + * @example + * var gcloud = require('gcloud')({ + * keyFilename: '/path/to/keyfile.json', + * projectId: 'my-project' + * }); + * + * var bigtable = gcloud.bigtable({ + * zone: 'us-central1-b', + * cluster: 'gcloud-node' + * }); + * + * //- + * //

Creating a Cluster

+ * // + * // Before you create your table, you first need to create a Bigtable Cluster + * // for the table to be served from. This can be done from either the + * // Google Cloud Platform Console or the `gcloud` cli tool. Please refer to + * // the + * // official Bigtable documentation for more information. + * //- + * + * //- + * //

Creating Tables

+ * // + * // After creating your cluster and enabling the Bigtable APIs, you are now + * // ready to create your table with {module:bigtable#createTable}. + * //- + * bigtable.createTable('prezzy', function(err, table) { + * // `table` is your newly created Table object. + * }); + * + * //- + * //

Creating Column Families

+ * // + * // Column families are used to group together various pieces of data within + * // your table. You can think of column families as a mechanism to categorize + * // all of your data. + * // + * // We can create a column family with {module:bigtable/table#createFamily}. + * //- + * var table = bigtable.table('prezzy'); + * + * table.createFamily('follows', function(err, family) { + * // `family` is your newly created Family object. + * }); + * + * //- + * //

Creating Rows

+ * // + * // New rows can be created within your table using + * // {module:bigtable/table#insert}. You must provide a unique key for each row + * // to be inserted, this key can then be used to retrieve your row at a later + * // time. + * // + * // With Bigtable, all columns have a unique id composed of a column family + * // and a column qualifier. In the example below `follows` is the column + * // family and `tjefferson` is the column qualifier. Together they could be + * // referred to as `follows:tjefferson`. + * //- + * var rows = [ + * { + * key: 'wmckinley', + * data: { + * follows: { + * tjefferson: 1 + * } + * } + * } + * ]; + * + * table.insert(rows, function(err) { + * if (!err) { + * // Your rows were successfully inserted. + * } + * }); + * + * //- + * //

Retrieving Rows

+ * // + * // If you're anticipating a large number of rows to be returned, we suggest + * // using the {module:bigtable/table#getRows} streaming API. + * //- + * table.getRows() + * .on('error', console.error) + * .on('data', function(row) { + * // `row` is a Row object. + * }); + * + * //- + * // If you're not anticpating a large number of results, a callback mode + * // is also available. + * //- + * var callback = function(err, rows) { + * // `rows` is an array of Row objects. + * }; + * + * table.getRows(callback); + * + * //- + * // A range of rows can be retrieved by providing `start` and `end` row keys. + * //- + * var options = { + * start: 'gwashington', + * end: 'wmckinley' + * }; + * + * table.getRows(options, callback); + * + * //- + * // Retrieve an individual row with {module:bigtable/row#get}. + * //- + * var row = table.row('alincoln'); + * + * row.get(function(err) { + * // `row.data` is now populated. + * }); + * + * //- + * //

Accessing Row Data

+ * // + * // When retrieving rows, upon success the `row.data` property will be + * // populated by an object. That object will contain additional objects + * // for each family in your table that the row has data for. + * // + * // By default, when retrieving rows, each column qualifier will provide you + * // with all previous versions of the data. So your `row.data` object could + * // resemble the following. + * //- + * console.log(row.data); + * // { + * // follows: { + * // wmckinley: [ + * // { + * // value: 1, + * // timestamp: 1466017315951 + * // }, { + * // value: 2, + * // timestamp: 1458619200000 + * // } + * // ] + * // } + * // } + * + * //- + * // The `timestamp` field can be used to order cells from newest to oldest. + * // If you only wish to retrieve the most recent version of the data, you + * // can specify the number of cells with a {module:bigtable/filter} object. + * //- + * var filter = [ + * { + * column: { + * cellLimit: 1 + * } + * } + * ]; + * + * table.getRows({ + * filter: filter + * }, callback); + * + * //- + * //

Deleting Row Data

+ * // + * // We can delete all of an individual row's cells using + * // {module:bigtable/row#delete}. + * //- + * var callback = function(err) { + * if (!err) { + * // All cells for this row were deleted successfully. + * } + * }; + * + * row.delete(callback); + * + * //- + * // To delete a specific set of cells, we can provide an array of + * // column families and qualifiers. + * //- + * var cells = [ + * 'follows:gwashington', + * 'traits' + * ]; + * + * row.delete(cells, callback); + * + * //- + * //

Deleting Rows

+ * // + * // If you wish to delete multiple rows entirely, we can do so with + * // {module:bigtable/table#deleteRows}. You can provide this method with a + * // row key prefix. + * //- + * var options = { + * prefix: 'gwash' + * }; + * + * table.deleteRows(options, function(err) { + * if (!err) { + * // Rows were deleted successfully. + * } + * }); + * + * //- + * // If you omit the prefix, you can delete all rows in your table. + * //- + * table.deleteRows(function(err) { + * if (!err) { + * // All rows were deleted successfully. + * } + * }); + */ +function Bigtable(options) { + if (!(this instanceof Bigtable)) { + options = util.normalizeArguments(this, options); + return new Bigtable(options); + } + + options = { + projectId: options.projectId, + zone: options.zone.name || options.zone, + cluster: options.cluster + }; + + this.clusterName = format( + 'projects/{projectId}/zones/{zone}/clusters/{cluster}', + options + ); + + var config = { + baseUrl: 'bigtable.googleapis.com', + service: 'bigtable', + apiVersion: 'v1', + protoServices: { + BigtableService: googleProtoFiles.bigtable.v1, + BigtableTableService: { + path: googleProtoFiles.bigtable.admin, + service: 'bigtable.admin.table' + } + }, + scopes: [ + 'https://www.googleapis.com/auth/bigtable.admin', + 'https://www.googleapis.com/auth/bigtable.data' + ] + }; + + GrpcService.call(this, config, options); +} + +nodeutil.inherits(Bigtable, GrpcService); + +/** + * Formats the full table name into a user friendly version. + * + * @private + * + * @param {string} name - The formatted Table name. + * @return {string} + * + * @example + * Bigtable.formatTableName_('projects/p/zones/z/clusters/c/tables/my-table'); + * // => 'my-table' + */ +Bigtable.formatTableName_ = function(name) { + if (name.indexOf('/') === -1) { + return name; + } + + var parts = name.split('/'); + return parts[parts.length - 1]; +}; + +/** + * Create a table on your Bigtable cluster. + * + * @resource [Designing Your Schema]{@link https://cloud.google.com/bigtable/docs/schema-design} + * @resource [Splitting Keys]{@link https://cloud.google.com/bigtable/docs/managing-tables#splits} + * + * @param {string} name - The name of the table. + * @param {object=} options - Table creation options. + * @param {string} options.operation - Operation used for table that has already + * been queued to be created. + * @param {string[]} options.splits - Initial + * [split keys](https://cloud.google.com/bigtable/docs/managing-tables#splits). + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request. + * @param {module:bigtable/table} callback.table - The newly created table. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * var callback = function(err, table, apiResponse) { + * // `table` is a Table object. + * }; + * + * bigtable.createTable('prezzy', callback); + * + * //- + * // Pre-split the table based on the row key to spread the load across + * // multiple Cloud Bigtable nodes. + * //- + * var options = { + * splits: ['10', '20'] + * }; + * + * bigtable.createTable('prezzy', options, callback); + */ +Bigtable.prototype.createTable = function(name, options, callback) { + var self = this; + + options = options || {}; + + if (is.function(options)) { + callback = options; + options = {}; + } + + var protoOpts = { + service: 'BigtableTableService', + method: 'createTable' + }; + + var reqOpts = { + name: this.clusterName, + tableId: name, + table: { + // The granularity at which timestamps are stored in the table. + // Currently only milliseconds is supported, so it's not configurable. + granularity: 0 + } + }; + + if (options.operation) { + reqOpts.table.currentOperation = options.operation; + } + + if (options.splits) { + reqOpts.initialSplitKeys = options.splits; + } + + this.request(protoOpts, reqOpts, function(err, resp) { + if (err) { + callback(err, null, resp); + return; + } + + var table = self.table(resp.name); + table.metadata = resp; + + callback(null, table, resp); + }); +}; + +/** + * Get Table objects for all the tables in your Bigtable cluster. + * + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request. + * @param {module:bigtable/table[]} callback.tables - List of all Tables. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * bigtable.getTables(function(err, tables) { + * if (!err) { + * // `tables` is an array of Table objects. + * } + * }); + */ +Bigtable.prototype.getTables = function(callback) { + var self = this; + + var protoOpts = { + service: 'BigtableTableService', + method: 'listTables' + }; + + var reqOpts = { + name: this.clusterName + }; + + this.request(protoOpts, reqOpts, function(err, resp) { + if (err) { + callback(err, null, resp); + return; + } + + var tables = resp.tables.map(function(metadata) { + var name = Bigtable.formatTableName_(metadata.name); + var table = self.table(name); + + table.metadata = metadata; + return table; + }); + + callback(null, tables, resp); + }); +}; + +/** + * Get a reference to a Bigtable table. + * + * @param {string} name - The name of the table. + * @return {module:bigtable/table} + * + * @example + * var table = bigtable.table('presidents'); + */ +Bigtable.prototype.table = function(name) { + return new Table(this, name); +}; + +module.exports = Bigtable; diff --git a/lib/bigtable/mutation.js b/lib/bigtable/mutation.js new file mode 100644 index 00000000000..c367fb40d37 --- /dev/null +++ b/lib/bigtable/mutation.js @@ -0,0 +1,332 @@ +/*! + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /*! + * @module bigtable/mutation + */ + +'use strict'; + +var is = require('is'); +var arrify = require('arrify'); +var Int64 = require('node-int64'); + +/** + * Formats table mutations to be in the expected proto format. + * + * @constructor + * @alias module:bigtable/mutation + * + * @example + * var mutation = new Mutation({ + * key: 'gwashington', + * method: 'insert', + * data: { + * jadams: 1 + * } + * }); + */ +function Mutation(mutation) { + this.key = mutation.key; + this.method = mutation.method; + this.data = mutation.data; +} + +/** + * Mutation methods + * + * INSERT => setCell + * DELETE => deleteFrom* + */ +var methods = Mutation.methods = { + INSERT: 'insert', + DELETE: 'delete' +}; + +/** + * Parses "bytes" returned from proto service. + * + * @param {string} bytes - Base64 encoded string. + * @return {string} + */ +Mutation.convertFromBytes = function(bytes) { + var buf = new Buffer(bytes, 'base64'); + var num = new Int64(buf).toNumber(); + + if (!isNaN(num) && isFinite(num)) { + return num; + } + + return buf.toString(); +}; + +/** + * Converts data into a buffer for proto service. + * + * @param {string} data - The data to be sent. + * @return {buffer} + */ +Mutation.convertToBytes = function(data) { + if (is.number(data)) { + return new Int64(data).toBuffer(); + } + + try { + return new Buffer(data); + } catch (e) { + return data; + } +}; + +/** + * Takes date objects and creates a time range. + * + * @param {date} start - The start date. + * @param {date} end - The end date. + * @return {object} + */ +Mutation.createTimeRange = function(start, end) { + var range = {}; + + if (is.date(start)) { + range.startTimestampMicros = start.getTime(); + } + + if (is.date(end)) { + range.endTimestampMicros = end.getTime(); + } + + return range; +}; + +/** + * Formats an `insert` mutation to what the proto service expects. + * + * @param {object} data - The entity data. + * @return {object[]} + * + * @example + * Mutation.encodeSetCell({ + * follows: { + * gwashington: 1, + * alincoln: 1 + * } + * }); + * // [ + * // { + * // setCell: { + * // familyName: 'follows', + * // columnQualifier: 'gwashington', // as buffer + * // timestampMicros: -1, // -1 means to use the server time + * // value: 1 // as buffer + * // } + * // }, { + * // setCell: { + * // familyName: 'follows', + * // columnQualifier: 'alincoln', // as buffer + * // timestampMicros: -1, + * // value: 1 // as buffer + * // } + * // } + * // ] + */ +Mutation.encodeSetCell = function(data) { + var mutations = []; + + Object.keys(data).forEach(function(familyName) { + var family = data[familyName]; + + Object.keys(family).forEach(function(cellName) { + var cell = family[cellName]; + + if (!is.object(cell)) { + cell = { + value: cell + }; + } + + var timestamp = cell.timestamp; + + if (is.date(timestamp)) { + timestamp = timestamp.getTime(); + } + + var setCell = { + familyName: familyName, + columnQualifier: Mutation.convertToBytes(cellName), + timestampMicros: timestamp || -1, + value: Mutation.convertToBytes(cell.value) + }; + + mutations.push({ setCell: setCell }); + }); + }); + + return mutations; +}; + +/** + * Formats a `delete` mutation to what the proto service expects. Depending + * on what data is supplied to this method, it will return an object that can + * will do one of the following: + * + * * Delete specific cells from a column. + * * Delete all cells contained with a specific family. + * * Delete all cells from an entire rows. + * + * @param {object} data - The entry data. + * @return {object} + * + * @example + * Mutation.encodeDelete([ + * 'follows:gwashington' + * ]); + * // { + * // deleteFromColumn: { + * // familyName: 'follows', + * // columnQualifier: 'gwashington', // as buffer + * // timeRange: null // optional + * // } + * // } + * + * Mutation.encodeDelete([ + * 'follows' + * ]); + * // { + * // deleteFromFamily: { + * // familyName: 'follows' + * // } + * // } + * + * Mutation.encodeDelete(); + * // { + * // deleteFromRow: {} + * // } + * + * //- + * // It's also possible to specify a time range when deleting specific columns. + * //- + * Mutation.encodeDelete([ + * { + * column: 'follows:gwashington', + * time: { + * start: new Date('March 21, 2000'), + * end: new Date('March 21, 2001') + * } + * } + * ]); + */ +Mutation.encodeDelete = function(data) { + if (!data) { + return [{ + deleteFromRow: {} + }]; + } + + return arrify(data).map(function(mutation) { + if (is.string(mutation)) { + mutation = { + column: mutation + }; + } + + var column = Mutation.parseColumnName(mutation.column); + + if (!column.qualifier) { + return { + deleteFromFamily: { + familyName: column.family + } + }; + } + + var timeRange; + + if (mutation.time) { + timeRange = Mutation.createTimeRange( + mutation.time.start, + mutation.time.end + ); + } + + return { + deleteFromColumn: { + familyName: column.family, + columnQualifier: Mutation.convertToBytes(column.qualifier), + timeRange: timeRange + } + }; + }); +}; + +/** + * Creates a new Mutation object and returns the proto JSON form. + * + * @param {object} entry - The entity data. + * @return {object} + */ +Mutation.parse = function(mutation) { + if (!(mutation instanceof Mutation)) { + mutation = new Mutation(mutation); + } + + return mutation.toProto(); +}; + +/** + * Parses a column name into an object. + * + * @param {string} column - The column name. + * @return {object} + * + * @example + * Mutation.parseColumnName('follows:gwashington'); + * // { + * // family: 'follows', + * // qualifier: 'gwashington' + * // } + */ +Mutation.parseColumnName = function(column) { + var parts = column.split(':'); + + return { + family: parts[0], + qualifier: parts[1] + }; +}; + +/** + * Converts the mutation object into proto friendly JSON. + * + * return {object} + */ +Mutation.prototype.toProto = function() { + var mutation = {}; + + if (this.key) { + mutation.rowKey = Mutation.convertToBytes(this.key); + } + + if (this.method === methods.INSERT) { + mutation.mutations = Mutation.encodeSetCell(this.data); + } else if (this.method === methods.DELETE) { + mutation.mutations = Mutation.encodeDelete(this.data); + } + + return mutation; +}; + +module.exports = Mutation; diff --git a/lib/bigtable/row.js b/lib/bigtable/row.js new file mode 100644 index 00000000000..3514a8ba634 --- /dev/null +++ b/lib/bigtable/row.js @@ -0,0 +1,750 @@ +/*! + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /*! + * @module bigtable/row + */ + +'use strict'; + +var nodeutil = require('util'); +var is = require('is'); +var extend = require('extend'); +var arrify = require('arrify'); +var flatten = require('lodash.flatten'); +var createErrorClass = require('create-error-class'); +var dotProp = require('dot-prop'); + +/** + * @type {module:bigtable/mutation} + * @private + */ +var Mutation = require('./mutation.js'); + +/** + * @type {module:bigtable/filter} + * @private + */ +var Filter = require('./filter.js'); + +/** + * @type {module:common/grpcServiceObject} + * @private + */ +var GrpcServiceObject = require('../common/grpc-service-object.js'); + +/** + * @private + */ +var RowError = createErrorClass('RowError', function(row) { + this.message = 'Unknown row: ' + row + '.'; +}); + +/** + * Create a Row object to interact with your table rows. + * + * @constructor + * @alias module:bigtable/row + * + * @example + * var gcloud = require('gcloud'); + * + * var bigtable = gcloud.bigtable({ + * keyFilename: '/path/to/keyfile.json', + * projectId: 'grape-spaceship-123', + * cluster: 'gcloud-node', + * zone: 'us-central1-b' + * }); + * + * var table = bigtable.table('prezzy'); + * var row = table.row('gwashington'); + */ +function Row(table, name) { + var methods = { + + /** + * Check if the table row exists. + * + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {boolean} callback.exists - Whether the row exists or not. + * + * @example + * row.exists(function(err, exists) {}); + */ + exists: true + }; + + var config = { + parent: table, + methods: methods, + id: name + }; + + GrpcServiceObject.call(this, config); + + this.data = {}; +} + +nodeutil.inherits(Row, GrpcServiceObject); + +/** + * Formats the row chunks into friendly format. Chunks contain 3 properties: + * + * `rowContents` - The row contents, this essentially is all data pertaining + * to a single family. + * + * `commitRow` - This is a boolean telling us the all previous chunks for this + * row are ok to consume. + * + * `resetRow` - This is a boolean telling us that all the previous chunks are to + * be discarded. + * + * @private + * + * @param {chunk[]} chunks - The list of chunks. + * + * @example + * var chunks = [ + * { + * rowContents: { + * name: 'follows', + * columns: [ + * { + * qualifier: 'gwashington', + * cells: [ + * { + * value: 1 + * } + * ] + * } + * ] + * } + * }, { + * resetRow: true + * }, { + * rowContents: { + * name: 'follows', + * columns: [ + * { + * qualifier: 'gwashington', + * cells: [ + * { + * value: 2 + * } + * ] + * } + * ] + * } + * }, { + * commitRow: true + * } + * ]; + * + * Row.formatChunks_(chunks); + * // { + * // follows: { + * // gwashington: [ + * // { + * // value: 2 + * // } + * // ] + * // } + * // } + */ +Row.formatChunks_ = function(chunks) { + var families = []; + var chunkList = []; + + chunks.forEach(function(chunk) { + if (chunk.resetRow) { + chunkList = []; + } + + if (chunk.rowContents) { + chunkList.push(chunk.rowContents); + } + + if (chunk.commitRow) { + families = families.concat(chunkList); + chunkList = []; + } + }); + + return Row.formatFamilies_(families); +}; + +/** + * Formats a rowContents object into friendly format. + * + * @private + * + * @param {object[]} families - The row families. + * + * @example + * var families = [ + * { + * name: 'follows', + * columns: [ + * { + * qualifier: 'gwashington', + * cells: [ + * { + * value: 2 + * } + * ] + * } + * ] + * } + * ]; + * + * Row.formatFamilies_(families); + * // { + * // follows: { + * // gwashington: [ + * // { + * // value: 2 + * // } + * // ] + * // } + * // } + */ +Row.formatFamilies_ = function(families) { + var data = {}; + + families.forEach(function(family) { + var familyData = data[family.name] = {}; + + family.columns.forEach(function(column) { + var qualifier = Mutation.convertFromBytes(column.qualifier); + + familyData[qualifier] = column.cells.map(function(cell) { + return { + value: Mutation.convertFromBytes(cell.value), + timestamp: cell.timestampMicros, + labels: cell.labels + }; + }); + }); + }); + + return data; +}; + +/** + * Create a new row in your table. + * + * @param {object=} entry - An entry. See {module:bigtable/table#insert}. + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {module:bigtable/row} callback.row - The newly created row object. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * var callback = function(err, apiResponse) { + * if (!err) { + * // Row successfully created + * } + * }; + * + * row.create(callback); + * + * //- + * // Optionally, you can supply entry data. + * //- + * row.create({ + * follows: { + * alincoln: 1 + * } + * }, callback); + */ +Row.prototype.create = function(entry, callback) { + var self = this; + + if (is.function(entry)) { + callback = entry; + entry = {}; + } + + entry = { + key: this.id, + data: entry, + method: Mutation.methods.INSERT + }; + + this.parent.mutate(entry, function(err, apiResponse) { + if (err) { + callback(err, null, apiResponse); + return; + } + + callback(null, self, apiResponse); + }); +}; + +/** + * Update a row with rules specifying how the row's contents are to be + * transformed into writes. Rules are applied in order, meaning that earlier + * rules will affect the results of later ones. + * + * @param {object|object[]} rules - The rules to apply to this row. + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * //- + * // Add an increment amount to an existing value, if the targeted cell is + * // unset, it will be treated as containing a zero. + * //- + * var callback = function(err, apiResponse) { + * if (!err) { + * // The rules have successfully been applied. + * } + * }; + * + * row.createRules([ + * { + * column: 'follows:gwashington', + * increment: 1 + * } + * ], callback); + * + * //- + * // You can also create a rule that will append data to an existing value. + * // If the targeted cell is unset, it will be treated as a containing an + * // empty string. + * //- + * row.createRules([ + * { + * column: 'follows:alincoln', + * append: ' Honest Abe!' + * } + * ], callback); + */ +Row.prototype.createRules = function(rules, callback) { + rules = arrify(rules).map(function(rule) { + var column = Mutation.parseColumnName(rule.column); + var ruleData = { + familyName: column.family, + columnQualifier: Mutation.convertToBytes(column.qualifier) + }; + + if (rule.append) { + ruleData.appendValue = Mutation.convertToBytes(rule.append); + } + + if (rule.increment) { + ruleData.incrementAmount = rule.increment; + } + + return ruleData; + }); + + var grpcOpts = { + service: 'BigtableService', + method: 'readModifyWriteRow' + }; + + var reqOpts = { + tableName: this.parent.id, + rowKey: Mutation.convertToBytes(this.id), + rules: rules + }; + + this.request(grpcOpts, reqOpts, callback); +}; + +/** + * Mutates a row atomically based on the output of a filter. Depending on + * whether or not any results are yielded, either the `onMatch` or `onNoMatch` + * callback will be executed. + * + * @param {module:bigtable/filter} filter - Filter ot be applied to the contents + * of the row. + * @param {?object[]} onMatch - A list of entries to be ran if a match is found. + * @param {object[]=} onNoMatch - A list of entries to be ran if no matches are + * found. + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {boolean} callback.matched - Whether a match was found or not. + * + * @example + * var callback = function(err, matched) { + * if (!err) { + * // `matched` will let us know if a match was found or not. + * } + * }; + * + * var filter = [ + * { + * family: 'follows' + * }, { + * column: 'alincoln', + * }, { + * value: 1 + * } + * ]; + * + * var entries = [ + * { + * method: 'insert', + * data: { + * follows: { + * jadams: 1 + * } + * } + * } + * ]; + * + * row.filter(filter, entries, callback); + * + * //- + * // Optionally, you can pass in an array of entries to be ran in the event + * // that a match is not made. + * //- + * row.filter(filter, null, entries, callback); + */ +Row.prototype.filter = function(filter, onMatch, onNoMatch, callback) { + var grpcOpts = { + service: 'BigtableService', + method: 'checkAndMutateRow' + }; + + if (is.function(onNoMatch)) { + callback = onNoMatch; + onNoMatch = []; + } + + var reqOpts = { + tableName: this.parent.id, + rowKey: Mutation.convertToBytes(this.id), + predicateFilter: Filter.parse(filter), + trueMutations: createFlatMutationsList(onMatch), + falseMutations: createFlatMutationsList(onNoMatch) + }; + + this.request(grpcOpts, reqOpts, function(err, apiResponse) { + if (err) { + callback(err, null, apiResponse); + return; + } + + callback(null, apiResponse.predicateMatched, apiResponse); + }); + + function createFlatMutationsList(entries) { + entries = arrify(entries).map(function(entry) { + return Mutation.parse(entry).mutations; + }); + + return flatten(entries); + } +}; + +/** + * Deletes all cells in the row. + * + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * row.delete(function(err, apiResponse) {}); + */ +Row.prototype.delete = function(callback) { + var mutation = { + key: this.id, + method: Mutation.methods.DELETE + }; + + this.parent.mutate(mutation, callback); +}; + +/** + * Delete specified cells from the row. See {module:bigtable/table#mutate}. + * + * @param {string[]} columns - Column names for the cells to be deleted. + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * //- + * // Delete individual cells. + * //- + * var callback = function(err, apiResponse) { + * if (!err) { + * // Cells were successfully deleted. + * } + * }; + * + * row.deleteCells([ + * 'follows:gwashington' + * ], callback); + * + * //- + * // Delete all cells within a family. + * //- + * row.deleteCells([ + * 'follows', + * ], callback) + */ +Row.prototype.deleteCells = function(columns, callback) { + var mutation = { + key: this.id, + data: arrify(columns), + method: Mutation.methods.DELETE + }; + + this.parent.mutate(mutation, callback); +}; + +/** + * Get the row data. See {module:bigtable/table#getRows}. + * + * @param {string[]=} columns - List of specific columns to retrieve. + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {module:bigtable/row} callback.row - The updated Row object. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * //- + * // Use this method to grab an entire row + * //- + * var callback = function(err, row, apiResponse) { + * if (!err) { + * // `row.cells` has been updated. + * } + * }; + * + * row.get(callback); + * + * //- + * // Or pass in an array of column names to populate specific cells. + * // Under the hood this will create an interleave filter. + * //- + * row.get([ + * 'follows:gwashington', + * 'follows:alincoln' + * ], callback); + */ +Row.prototype.get = function(columns, callback) { + var self = this; + + if (is.function(columns)) { + callback = columns; + columns = []; + } + + var filter; + + columns = arrify(columns); + + if (columns.length) { + var filters = columns + .map(Mutation.parseColumnName) + .map(function(column) { + var filters = [{ family: column.family }]; + + if (column.qualifier) { + filters.push({ column: column.qualifier }); + } + + return filters; + }); + + if (filters.length > 1) { + filter = [{ + interleave: filters + }]; + } else { + filter = filters[0]; + } + } + + var reqOpts = { + key: this.id, + filter: filter + }; + + this.parent.getRows(reqOpts, function(err, rows, apiResponse) { + if (err) { + callback(err, null, apiResponse); + return; + } + + var row = rows[0]; + + if (!row) { + err = new RowError(self.id); + callback(err, null, apiResponse); + return; + } + + extend(true, self.data, row.data); + + // If the user specifies column names, we'll return back the row data we + // received. Otherwise, we'll return the row itself in a typical + // GrpcServiceObject#get fashion. + callback(null, columns.length ? row.data : self, apiResponse); + }); +}; + +/** + * Get the row's metadata. + * + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {object} callback.metadata - The row's metadata. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * row.getMetadata(function(err, metadata, apiResponse) {}); + */ +Row.prototype.getMetadata = function(callback) { + this.get(function(err, row, resp) { + if (err) { + callback(err, null, resp); + return; + } + + callback(null, row.metadata, resp); + }); +}; + +/** + * Increment a specific column within the row. If the column does not + * exist, it is automatically initialized to 0 before being incremented. + * + * @param {string} column - The column we are incrementing a value in. + * @param {number=} value - The amount to increment by, defaults to 1. + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {number} callback.value - The updated value of the column. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * var callback = function(err, value, apiResponse) { + * if (!err) { + * // `value` is the value of the updated column. + * } + * }; + * + * row.increment('follows:gwashington', callback) + * + * //- + * // Specify a custom amount to increment the column by. + * //- + * row.increment('follows:gwashington', 2, callback); + * + * //- + * // To decrement a column, simply supply a negative value. + * //- + * row.increment('follows:gwashington', -1, callback); + */ +Row.prototype.increment = function(column, value, callback) { + if (is.function(value)) { + callback = value; + value = 1; + } + + var reqOpts = { + column: column, + increment: value + }; + + this.createRules(reqOpts, function(err, apiResponse) { + if (err) { + callback(err, null, apiResponse); + return; + } + + var data = Row.formatFamilies_(apiResponse.families); + var value = dotProp.get(data, column.replace(':', '.'))[0].value; + + callback(null, value, apiResponse); + }); +}; + +/** + * Update the row cells. + * + * @param {string|object} key - Either a column name or an entry + * object to be inserted into the row. See {module:bigtable/table#insert}. + * @param {*=} value - This can be omitted if using entry object. + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * //- + * // Update a single cell. + * //- + * var callback = function(err, apiResponse) { + * if (!err) { + * // The row has been successfully updated. + * } + * }; + * + * row.save('follows:jadams', 1, callback); + * + * //- + * // Or update several cells at once. + * //- + * row.save({ + * follows: { + * jadams: 1, + * wmckinley: 1 + * } + * }, callback); + */ +Row.prototype.save = function(key, value, callback) { + var rowData; + + if (is.string(key)) { + var column = Mutation.parseColumnName(key); + + rowData = {}; + rowData[column.family] = {}; + rowData[column.family][column.qualifier] = value; + } else { + rowData = key; + callback = value; + } + + var mutation = { + key: this.id, + data: rowData, + method: Mutation.methods.INSERT + }; + + this.parent.mutate(mutation, callback); +}; + +module.exports = Row; +module.exports.RowError = RowError; diff --git a/lib/bigtable/table.js b/lib/bigtable/table.js new file mode 100644 index 00000000000..48c7beeac5c --- /dev/null +++ b/lib/bigtable/table.js @@ -0,0 +1,857 @@ +/*! + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/*! + * @module bigtable/table + */ + +'use strict'; + +var nodeutil = require('util'); +var arrify = require('arrify'); +var is = require('is'); +var propAssign = require('prop-assign'); +var through = require('through2'); +var concat = require('concat-stream'); +var pumpify = require('pumpify'); +var flatten = require('lodash.flatten'); + +/** + * @type {module:bigtable/family} + * @private + */ +var Family = require('./family.js'); + +/** + * @type {module:bigtable/row} + * @private + */ +var Row = require('./row.js'); + +/** + * @type {module:bigtable/filter} + * @private + */ +var Filter = require('./filter.js'); + +/** + * @type {module:bigtable/mutation} + * @private + */ +var Mutation = require('./mutation.js'); + +/** + * @type {module:common/grpcServiceObject} + * @private + */ +var GrpcServiceObject = require('../common/grpc-service-object.js'); + +/** + * Create a Table object to interact with a Google Cloud Bigtable table. + * + * @constructor + * @alias module:bigtable/table + * + * @param {string} name - Name of the table. + * + * @example + * var gcloud = require('gcloud'); + * + * var bigtable = gcloud.bigtable({ + * keyFilename: '/path/to/keyfile.json', + * projectId: 'grape-spaceship-123', + * cluster: 'gcloud-node', + * zone: 'us-central1-b' + * }); + * + * var table = bigtable.table('prezzy'); + */ +function Table(bigtable, name) { + var id = Table.formatName_(bigtable.clusterName, name); + + var methods = { + + /** + * Create a table. + * + * @param {object=} options - See {module:bigtable#createTable}. + * + * @example + * table.create(function(err, table, apiResponse) { + * if (!err) { + * // The table was created successfully. + * } + * }); + */ + create: true, + + /** + * Delete the table. + * + * @param {function=} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * table.delete(function(err, apiResponse) {}); + */ + delete: { + protoOpts: { + service: 'BigtableTableService', + method: 'deleteTable' + }, + reqOpts: { + name: id + } + }, + + /** + * Check if a table exists. + * + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {boolean} callback.exists - Whether the table exists or not. + * + * @example + * table.exists(function(err, exists) {}); + */ + exists: true, + + /** + * Get a table if it exists. + * + * You may optionally use this to "get or create" an object by providing an + * object with `autoCreate` set to `true`. Any extra configuration that is + * normally required for the `create` method must be contained within this + * object as well. + * + * @param {options=} options - Configuration object. + * @param {boolean} options.autoCreate - Automatically create the object if + * it does not exist. Default: `false` + * + * @example + * table.get(function(err, table, apiResponse) { + * // The `table` data has been populated. + * }); + */ + get: true, + + /** + * Get the table's metadata. + * + * @param {function=} callback - The callback function. + * @param {?error} callback.err - An error returned while making this + * request. + * @param {object} callback.metadata - The table's metadata. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * table.getMetadata(function(err, metadata, apiResponse) {}); + */ + getMetadata: { + protoOpts: { + service: 'BigtableTableService', + method: 'getTable' + }, + reqOpts: { + name: id + } + } + }; + + var config = { + parent: bigtable, + id: id, + methods: methods, + createMethod: function(_, options, callback) { + bigtable.createTable(name, options, callback); + } + }; + + GrpcServiceObject.call(this, config); +} + +nodeutil.inherits(Table, GrpcServiceObject); + +/** + * Formats the table name to include the Bigtable cluster. + * + * @private + * + * @param {string} clusterName - The formatted cluster name. + * @param {string} name - The table name. + * + * @example + * Table.formatName_( + * 'projects/my-project/zones/my-zone/clusters/my-cluster', + * 'my-table' + * ); + * // 'projects/my-project/zones/my-zone/clusters/my-cluster/tables/my-table' + */ +Table.formatName_ = function(clusterName, name) { + if (name.indexOf('/') > -1) { + return name; + } + + return clusterName + '/tables/' + name; +}; + +/** + * Formats a row range into the desired proto format. + * + * @private + * + * @param {object} range - The range object. + * @param {string} range.start - The lower bound for the range. + * @param {string} range.end - The upper bound for the range. + * @return {object} + * + * @example + * Table.formatRowRange_({ + * start: 'gwashington', + * end: 'alincoln' + * }); + * // { + * // startKey: new Buffer('gwashington'), + * // endKey: new Buffer('alincoln') + * // } + */ +Table.formatRowRange_ = function(range) { + var rowRange = {}; + + if (range.start) { + rowRange.startKey = Mutation.convertToBytes(range.start); + } + + if (range.end) { + rowRange.endKey = Mutation.convertToBytes(range.end); + } + + return rowRange; +}; + +/** + * Create a column family. + * + * Optionally you can send garbage collection rules and expressions when + * creating a family. Garbage collection executes opportunistically in the + * background, so it's possible for reads to return a cell even if it + * matches the active expression for its family. + * + * @resource [Garbage Collection Proto Docs]{@link https://github.com/googleapis/googleapis/blob/master/google/bigtable/admin/table/v1/bigtable_table_data.proto#L59} + * + * @param {string} name - The name of column family. + * @param {string|object=} rule - Garbage collection rule. + * @param {object=} rule.age - Delete cells in a column older than the given + * age. Values must be at least 1 millisecond. + * @param {number} rule.versions - Maximum number of versions to delete cells + * in a column, except for the most recent. + * @param {boolean} rule.intersect - Cells to delete should match all rules. + * @param {boolean} rule.union - Cells to delete should match any of the rules. + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request. + * @param {module:bigtable/family} callback.family - The newly created Family. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * var callback = function(err, family, apiResponse) { + * // `family` is a Family object + * }; + * + * var rule = { + * age: { + * seconds: 0, + * nanos: 5000 + * }, + * versions: 3, + * union: true + * }; + * + * table.createFamily('follows', rule, callback); + * + * //- + * // Alternatively you can send a garbage collection expression. + * //- + * var expression = 'version() > 3 || (age() > 3d && version() > 1)'; + * + * table.createFamily('follows', expression, callback); + */ +Table.prototype.createFamily = function(name, rule, callback) { + var self = this; + + if (is.function(rule)) { + callback = rule; + rule = null; + } + + var grpcOpts = { + service: 'BigtableTableService', + method: 'createColumnFamily' + }; + + var reqOpts = { + name: this.id, + columnFamilyId: name + }; + + if (is.string(rule)) { + reqOpts.columnFamily = { + gcExpression: rule + }; + } else if (is.object(rule)) { + reqOpts.columnFamily = { + gcRule: Family.formatRule_(rule) + }; + } + + this.request(grpcOpts, reqOpts, function(err, resp) { + if (err) { + callback(err, null, resp); + return; + } + + var family = self.family(resp.name); + family.metadata = resp; + callback(null, family, resp); + }); +}; + +/** + * Delete all rows in the table, optionally corresponding to a particular + * prefix. + * + * @param {options=} options - Configuration object. + * @param {string} options.prefix - Row key prefix, when omitted all rows + * will be deleted. + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * //- + * // You can supply a prefix to delete all corresponding rows. + * //- + * var callback = function(err, apiResponse) { + * if (!err) { + * // Rows successfully deleted. + * } + * }; + * + * table.deleteRows({ + * prefix: 'alincoln' + * }, callback); + * + * //- + * // If you choose to omit the prefix, all rows in the table will be deleted. + * //- + * table.deleteRows(callback); + */ +Table.prototype.deleteRows = function(options, callback) { + if (is.function(options)) { + callback = options; + options = {}; + } + + var grpcOpts = { + service: 'BigtableTableService', + method: 'bulkDeleteRows' + }; + + var reqOpts = { + tableName: this.id + }; + + if (options.prefix) { + reqOpts.rowKeyPrefix = Mutation.convertToBytes(options.prefix); + } else { + reqOpts.deleteAllDataFromTable = true; + } + + this.request(grpcOpts, reqOpts, callback); +}; + +/** + * Get a reference to a Table Family. + * + * @param {string} name - The family name. + * @return {module:bigtable/family} + * + * @example + * var family = table.family('my-family'); + */ +Table.prototype.family = function(name) { + return new Family(this, name); +}; + +/** + * Get Family objects for all the column familes in your table. + * + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request. + * @param {module:bigtable/family[]} callback.families - The list of families. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * table.getFamilies(function(err, families, apiResponse) { + * // `families` is an array of Family objects. + * }); + */ +Table.prototype.getFamilies = function(callback) { + var self = this; + + this.getMetadata(function(err, resp) { + if (err) { + callback(err, null, resp); + return; + } + + var families = Object.keys(resp.columnFamilies).map(function(familyId) { + var family = self.family(familyId); + family.metadata = resp.columnFamilies[familyId]; + return family; + }); + + callback(null, families, resp); + }); +}; + +/** + * Get Row objects for the rows currently in your table. + * + * @param {options=} options - Configuration object. + * @param {string} options.key - An individual row key. + * @param {string[]} options.keys - A list of row keys. + * @param {string} options.start - Start value for key range. + * @param {string} options.end - End value for key range. + * @param {object[]} options.ranges - A list of key ranges. + * @param {module:bigtable/filter} options.filter - Row filters allow you to + * both make advanced queries and format how the data is returned. + * @param {boolean} options.interleave - Allow for interleaving. + * @param {number} options.limit - Maximum number of rows to be returned. + * @param {function=} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request. + * @param {module:bigtable/row[]} callback.rows - List of Row objects. + * + * @example + * //- + * // While this method does accept a callback, this is not recommended for + * // large datasets as it will buffer all rows before executing the callback. + * // Instead we recommend using the streaming API by simply omitting the + * // callback. + * //- + * var callback = function(err, rows) { + * if (!err) { + * // `rows` is an array of Row objects. + * } + * }; + * + * table.getRows(callback); + * + * //- + * // Specify a single row to be returned. + * //- + * table.getRows({ + * key: 'alincoln' + * }, callback); + * + * //- + * // Specify arbitrary keys for a non-contiguous set of rows. + * // The total size of the keys must remain under 1MB, after encoding. + * //- + * table.getRows({ + * keys: [ + * 'alincoln', + * 'gwashington' + * ] + * }, callback); + * + * //- + * // Specify a contiguous range of rows to read by supplying `start` and `end` + * // keys. + * // + * // If the `start` key is omitted, it is interpreted as an empty string. + * // If the `end` key is omitted, it is interpreted as infinity. + * //- + * table.getRows({ + * start: 'alincoln', + * end: 'gwashington' + * }, callback); + * + * //- + * // Specify multiple ranges. + * //- + * table.getRows({ + * ranges: [{ + * start: 'alincoln', + * end: 'gwashington' + * }, { + * start: 'tjefferson', + * end: 'jadams' + * }] + * }, callback); + * + * //- + * // By default, rows are read sequentially, producing results which are + * // guaranteed to arrive in increasing row order. Setting `interleave` to + * // true allows multiple rows to be interleaved in the response, which + * // increases throughput but breaks this guarantee and may force the client + * // to use more memory to buffer partially-received rows. + * //- + * table.getRows({ + * interleave: true + * }, callback); + * + * //- + * // Apply a {module:bigtable/filter} to the contents of the specified rows. + * //- + * table.getRows({ + * filter: [ + * { + * column: 'gwashington' + * }, { + * value: 1 + * } + * ] + * }, callback); + * + * //- + * // Get the rows from your table as a readable object stream. + * //- + * table.getRows() + * .on('error', console.error) + * .on('data', function(row) { + * // `row` is a Row object. + * }) + * .on('end', function() { + * // All rows retrieved. + * }); + * + * //- + * // If you anticipate many results, you can end a stream early to prevent + * // unnecessary processing. + * //- + * table.getRows() + * .on('data', function(row) { + * this.end(); + * }); + */ +Table.prototype.getRows = function(options, callback) { + var self = this; + + if (is.function(options)) { + callback = options; + options = {}; + } + + options = options || {}; + + var grpcOpts = { + service: 'BigtableService', + method: 'readRows' + }; + + var reqOpts = { + tableName: this.id, + objectMode: true + }; + + if (options.key) { + reqOpts.rowKey = Mutation.convertToBytes(options.key); + } else if (options.start || options.end) { + reqOpts.rowRange = Table.formatRowRange_(options); + } else if (options.keys || options.ranges) { + reqOpts.rowSet = {}; + + if (options.keys) { + reqOpts.rowSet.rowKeys = options.keys.map(Mutation.convertToBytes); + } + + if (options.ranges) { + reqOpts.rowSet.rowRanges = options.ranges.map(Table.formatRowRange_); + } + } + + if (options.filter) { + reqOpts.filter = Filter.parse(options.filter); + } + + if (options.interleave) { + reqOpts.allowRowInterleaving = options.interleave; + } + + if (options.limit) { + reqOpts.numRowsLimit = options.limit; + } + + var stream = pumpify.obj([ + this.requestStream(grpcOpts, reqOpts), + through.obj(function(rowData, enc, next) { + var row = self.row(Mutation.convertFromBytes(rowData.rowKey)); + + row.data = Row.formatChunks_(rowData.chunks); + next(null, row); + }) + ]); + + if (!is.function(callback)) { + return stream; + } + + stream + .on('error', callback) + .pipe(concat(function(rows) { + callback(null, rows); + })); +}; + +/** + * Insert or update rows in your table. + * + * @param {object|object[]} entries - List of entries to be inserted. + * See {module:bigtable/table#mutate}. + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * var entries = [ + * { + * key: 'alincoln', + * data: { + * follows: { + * gwashington: 1 + * } + * } + * } + * ]; + * + * table.insert(entries, function(err, apiResponse) {}); + * + * //- + * // By default whenever you insert new data, the server will capture a + * // timestamp of when your data was inserted. It's possible to provide a + * // date object to be used instead. + * //- + * var entries = [ + * { + * key: 'gwashington', + * data: { + * follows: { + * jadams: { + * value: 1, + * timestamp: new Date('March 22, 2016') + * } + * } + * } + * } + * ]; + * + * table.insert(entries, function(err, apiResponse) {}); + */ +Table.prototype.insert = function(entries, callback) { + entries = arrify(entries).map(propAssign('method', Mutation.methods.INSERT)); + + return this.mutate(entries, callback); +}; + +/** + * Apply a set of changes to be atomically applied to the specified row(s). + * Mutations are applied in order, meaning that earlier mutations can be masked + * by later ones. + * + * @param {object|object[]} entries - List of entities to be inserted or + * deleted. + * @param {function} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request. + * @param {object} callback.apiResponse - The full API response. + * + * @example + * //- + * // Insert entities. See {module:bigtable/table#insert} + * //- + * var callback = function(err, apiResponse) { + * if (!err) { + * // Mutations were successful. + * } + * }; + * + * var entries = [ + * { + * method: 'insert', + * key: 'gwashington', + * data: { + * follows: { + * jadams: 1 + * } + * } + * } + * ]; + * + * table.mutate(entries, callback) + * + * //- + * // Delete entities. See {module:bigtable/row#deleteCells} + * //- + * var entries = [ + * { + * method: 'delete', + * key: 'gwashington' + * } + * ]; + * + * table.mutate(entries, callback); + * + * //- + * // Delete specific columns within a row. + * //- + * var entries = [ + * { + * method: 'delete', + * key: 'gwashington', + * data: [ + * 'follows:jadams' + * ] + * } + * ]; + * + * table.mutate(entries, callback); + * + * //- + * // Mix and match mutations. This must contain at least one entry and at + * // most 100,000. + * //- + * var entries = [ + * { + * method: 'insert', + * key: 'alincoln', + * data: { + * follows: { + * gwashington: 1 + * } + * } + * }, { + * method: 'delete', + * key: 'jadams', + * data: [ + * 'follows:gwashington' + * ] + * } + * ]; + * + * table.mutate(entries, callback); + */ +Table.prototype.mutate = function(entries, callback) { + entries = flatten(arrify(entries)).map(Mutation.parse); + + var grpcOpts = { + service: 'BigtableService', + method: 'mutateRows' + }; + + var reqOpts = { + tableName: this.id, + entries: entries + }; + + this.request(grpcOpts, reqOpts, callback); +}; + +/** + * Get a reference to a table row. + * + * @param {string} key - The row key. + * @return {module:bigtable/row} + * + * @example + * var row = table.row('lincoln'); + */ +Table.prototype.row = function(key) { + return new Row(this, key); +}; + +/** + * Returns a sample of row keys in the table. The returned row keys will delimit + * contigous sections of the table of approximately equal size, which can be + * used to break up the data for distributed tasks like mapreduces. + * + * @param {function=} callback - The callback function. + * @param {?error} callback.err - An error returned while making this request. + * @param {object[]} callback.keys - The list of keys. + * + * @example + * table.sampleRowKeys(function(err, keys) { + * // keys = [ + * // { + * // key: '', + * // offset: '805306368' + * // }, + * // ... + * // ] + * }); + * + * //- + * // Get the keys from your table as a readable object stream. + * //- + * table.sampleRowKeys() + * .on('error', console.error) + * .on('data', function(key) { + * // Do something with the `key` object. + * }); + * + * //- + * // If you anticipate many results, you can end a stream early to prevent + * // unnecessary processing. + * //- + * table.sampleRowKeys() + * .on('data', function(key) { + * this.end(); + * }); + */ +Table.prototype.sampleRowKeys = function(callback) { + var grpcOpts = { + service: 'BigtableService', + method: 'sampleRowKeys' + }; + + var reqOpts = { + tableName: this.id, + objectMode: true + }; + + var stream = pumpify.obj([ + this.requestStream(grpcOpts, reqOpts), + through.obj(function(key, enc, next) { + next(null, { + key: key.rowKey, + offset: key.offsetBytes + }); + }) + ]); + + if (!is.function(callback)) { + return stream; + } + + stream + .on('error', callback) + .pipe(concat(function(keys) { + callback(null, keys); + })); +}; + +module.exports = Table; diff --git a/lib/common/grpc-service-object.js b/lib/common/grpc-service-object.js index 290352edb08..5c2992e3e27 100644 --- a/lib/common/grpc-service-object.js +++ b/lib/common/grpc-service-object.js @@ -99,7 +99,16 @@ GrpcServiceObject.prototype.setMetadata = function(metadata, callback) { * @private */ GrpcServiceObject.prototype.request = function(protoOpts, reqOpts, callback) { - this.parent.request(protoOpts, reqOpts, callback); + return this.parent.request(protoOpts, reqOpts, callback); +}; + +/** + * Patch a streaming request to the GrpcService object. + * + * @private + */ +GrpcServiceObject.prototype.requestStream = function(protoOpts, reqOpts) { + return this.parent.requestStream(protoOpts, reqOpts); }; module.exports = GrpcServiceObject; diff --git a/lib/common/grpc-service.js b/lib/common/grpc-service.js index 4d69778a90a..9c9de954361 100644 --- a/lib/common/grpc-service.js +++ b/lib/common/grpc-service.js @@ -27,6 +27,8 @@ var is = require('is'); var nodeutil = require('util'); var path = require('path'); var retryRequest = require('retry-request'); +var through = require('through2'); +var dotProp = require('dot-prop'); /** * @type {module:common/service} @@ -160,7 +162,6 @@ function GrpcService(config, options) { var apiVersion = config.apiVersion; var service = this.service = config.service; - var rootDir = googleProtoFiles('..'); this.activeServiceMap_ = new Map(); this.protos = {}; @@ -172,17 +173,11 @@ function GrpcService(config, options) { protoServices[service] = googleProtoFiles[service][apiVersion]; } - for (var protoService in protoServices) { - var protoFilePath = protoServices[protoService]; - var grpcOpts = { - binaryAsBase64: true, - convertFieldsToCamelCase: true - }; + for (var protoServiceName in protoServices) { + var protoService = this.loadProtoFile_( + protoServices[protoServiceName], config); - this.protos[protoService] = grpc.load({ - root: rootDir, - file: path.relative(rootDir, protoFilePath) - }, 'proto', grpcOpts).google[service][apiVersion]; + this.protos[protoServiceName] = protoService; } } @@ -205,13 +200,6 @@ GrpcService.prototype.request = function(protoOpts, reqOpts, callback) { } var self = this; - var proto; - - if (this.protos[protoOpts.service]) { - proto = this.protos[protoOpts.service]; - } else { - proto = this.protos[this.service]; - } if (!this.grpcCredentials) { // We must establish an authClient to give to grpc. @@ -224,6 +212,7 @@ GrpcService.prototype.request = function(protoOpts, reqOpts, callback) { self.grpcCredentials = credentials; self.request(protoOpts, reqOpts, callback); }); + return; } @@ -231,33 +220,20 @@ GrpcService.prototype.request = function(protoOpts, reqOpts, callback) { delete reqOpts.autoPaginate; delete reqOpts.autoPaginateVal; - var service = this.activeServiceMap_.get(protoOpts.service); - - if (!service) { - service = new proto[protoOpts.service]( - this.baseUrl, - this.grpcCredentials - ); - - this.activeServiceMap_.set(protoOpts.service, service); - } - + var service = this.getService_(protoOpts); var grpcOpts = {}; if (is.number(protoOpts.timeout)) { - grpcOpts.deadline = new Date(Date.now() + protoOpts.timeout); + grpcOpts.deadline = GrpcService.createDeadline_(protoOpts.timeout); } // Retains a reference to an error from the response. If the final callback is // executed with this as the "response", we return it to the user as an error. var respError; - retryRequest(null, { - shouldRetryFn: function(resp) { - return [429, 500, 502, 503].indexOf(resp.code) > -1; - }, - + var retryOpts = { retries: this.maxRetries, + shouldRetryFn: GrpcService.shouldRetryRequest_, // retry-request determines if it should retry from the incoming HTTP // response status. gRPC always returns an error proto message. We pass that @@ -268,20 +244,23 @@ GrpcService.prototype.request = function(protoOpts, reqOpts, callback) { service[protoOpts.method](reqOpts, grpcOpts, function(err, resp) { if (err) { - if (GRPC_ERROR_CODE_TO_HTTP[err.code]) { - respError = extend(err, GRPC_ERROR_CODE_TO_HTTP[err.code]); + respError = GrpcService.getError_(err); + + if (respError) { onResponse(null, respError); return; } - onResponse(err); + onResponse(err, resp); return; } onResponse(null, resp); }); } - }, function(err, resp) { + }; + + retryRequest(null, retryOpts, function(err, resp) { if (!err && resp === respError) { err = respError; resp = null; @@ -291,6 +270,78 @@ GrpcService.prototype.request = function(protoOpts, reqOpts, callback) { }); }; +/** + * Make an authenticated streaming request with gRPC. + * + * @param {object} protoOpts - The proto options. + * @param {string} protoOpts.service - The service git stat. + * @param {string} protoOpts.method - The method name. + * @param {number=} protoOpts.timeout - After how many milliseconds should the + * request cancel. + * @param {object} reqOpts - The request options. + */ +GrpcService.prototype.requestStream = function(protoOpts, reqOpts) { + if (global.GCLOUD_SANDBOX_ENV) { + return through.obj(); + } + + var self = this; + + if (!protoOpts.stream) { + protoOpts.stream = through.obj(); + } + + var stream = protoOpts.stream; + + if (!this.grpcCredentials) { + // We must establish an authClient to give to grpc. + this.getGrpcCredentials_(function(err, credentials) { + if (err) { + stream.destroy(err); + return; + } + + self.grpcCredentials = credentials; + self.requestStream(protoOpts, reqOpts); + }); + + return stream; + } + + var objectMode = !!reqOpts.objectMode; + delete reqOpts.objectMode; + + var service = this.getService_(protoOpts); + var grpcOpts = {}; + + if (is.number(protoOpts.timeout)) { + grpcOpts.deadline = GrpcService.createDeadline_(protoOpts.timeout); + } + + var retryOpts = { + retries: this.maxRetries, + objectMode: objectMode, + shouldRetryFn: GrpcService.shouldRetryRequest_, + + request: function() { + return service[protoOpts.method](reqOpts, grpcOpts) + .on('status', function(status) { + var grcpStatus = GrpcService.getError_(status); + + this.emit('response', grcpStatus || status); + }); + } + }; + + return retryRequest(null, retryOpts) + .on('error', function(err) { + var grpcError = GrpcService.getError_(err); + + stream.destroy(grpcError || err); + }) + .pipe(stream); +}; + /** * Decode a protobuf Struct's value. * @@ -383,6 +434,46 @@ GrpcService.encodeValue_ = function(value, options) { return convertedValue; }; +/** + * Creates a deadline. + * + * @private + * + * @param {number} timeout - Timeout in miliseconds. + * @return {date} deadline - The deadline in Date object form. + */ +GrpcService.createDeadline_ = function(timeout) { + return new Date(Date.now() + timeout); +}; + +/** + * Checks for a grpc error code and extends the Error object with additional + * information. + * + * @private + * + * @param {error} err - The original request error. + * @return {error|null} + */ +GrpcService.getError_ = function(err) { + if (GRPC_ERROR_CODE_TO_HTTP[err.code]) { + return extend(true, {}, err, GRPC_ERROR_CODE_TO_HTTP[err.code]); + } + return null; +}; + +/** + * Function to decide whether or not a request retry could occur. + * + * @private + * + * @param {object} response - The request response. + * @return {boolean} shouldRetry + */ +GrpcService.shouldRetryRequest_ = function(response) { + return [429, 500, 502, 503].indexOf(response.code) > -1; +}; + /** * Convert an object to a struct. * @@ -514,5 +605,72 @@ GrpcService.prototype.getGrpcCredentials_ = function(callback) { }); }; +/** + * Loads a proto file, useful when handling multiple proto files/services + * within a single instance of GrpcService. + * + * @private + * + * @param {object} protoConfig - The proto specific configs for this file. + * @param {object} config - The base config for the GrpcService. + * @return {object} protoObject - The loaded proto object. + */ +GrpcService.prototype.loadProtoFile_ = function(protoConfig, config) { + var rootDir = googleProtoFiles('..'); + + var grpcOpts = { + binaryAsBase64: true, + convertFieldsToCamelCase: true + }; + + if (is.string(protoConfig)) { + protoConfig = { + path: protoConfig + }; + } + + var services = grpc.load({ + root: rootDir, + file: path.relative(rootDir, protoConfig.path) + }, 'proto', grpcOpts); + + var serviceName = protoConfig.service || config.service; + var apiVersion = protoConfig.apiVersion || config.apiVersion; + var service = dotProp.get(services.google, serviceName); + + return service[apiVersion]; +}; + +/** + * Retrieves the service object used to make the grpc requests. + * + * @private + * + * @param {object} protoOpts - The proto options. + * @return {object} service - The proto service. + */ +GrpcService.prototype.getService_ = function(protoOpts) { + var proto; + + if (this.protos[protoOpts.service]) { + proto = this.protos[protoOpts.service]; + } else { + proto = this.protos[this.service]; + } + + var service = this.activeServiceMap_.get(protoOpts.service); + + if (!service) { + service = new proto[protoOpts.service]( + this.baseUrl, + this.grpcCredentials + ); + + this.activeServiceMap_.set(protoOpts.service, service); + } + + return service; +}; + module.exports = GrpcService; module.exports.GRPC_ERROR_CODE_TO_HTTP = GRPC_ERROR_CODE_TO_HTTP; diff --git a/lib/index.js b/lib/index.js index 21accae958b..6f72b793eb7 100644 --- a/lib/index.js +++ b/lib/index.js @@ -48,6 +48,26 @@ var apis = { */ bigquery: require('./bigquery'), + /** + * [Cloud Bigtable](https://cloud.google.com/bigtable/) is Google's NoSQL Big + * Data database service. It's the same database that powers many core Google + * services, including Search, Analytics, Maps, and Gmail. + * + * @type {module:bigtable} + * + * @return {module:bigtable} + * + * @example + * var gcloud = require('gcloud'); + * var bigtable = gcloud.bigtable({ + * projectId: 'grape-spaceship-123', + * keyFilename: '/path/to/keyfile.json', + * zone: 'us-central1-b', + * cluster: 'gcloud-node' + * }); + */ + bigtable: require('./bigtable'), + /** * With [Compute Engine](https://cloud.google.com/compute/), you can run * large-scale workloads on virtual machines hosted on Google's diff --git a/lib/pubsub/subscription.js b/lib/pubsub/subscription.js index 074afadd4be..bd48ec50679 100644 --- a/lib/pubsub/subscription.js +++ b/lib/pubsub/subscription.js @@ -651,7 +651,7 @@ Subscription.prototype.listenForEvents_ = function() { if (event === 'message' && --self.messageListeners === 0) { self.closed = true; - if (self.activeRequest_) { + if (self.activeRequest_ && self.activeRequest_.cancel) { self.activeRequest_.cancel(); } } diff --git a/package.json b/package.json index 64b0c2e6611..9b4dadf1abe 100644 --- a/package.json +++ b/package.json @@ -96,6 +96,7 @@ "concat-stream": "^1.5.0", "create-error-class": "^2.0.1", "dns-zonefile": "0.1.18", + "dot-prop": "^2.4.0", "duplexify": "^3.2.0", "ent": "^2.2.0", "extend": "^3.0.0", @@ -110,12 +111,13 @@ "methmeth": "^1.0.0", "mime-types": "^2.0.8", "modelo": "^4.2.0", + "node-int64": "^0.4.0", "once": "^1.3.1", "prop-assign": "^1.0.0", "propprop": "^0.3.0", "pumpify": "^1.3.3", "request": "^2.70.0", - "retry-request": "^1.2.3", + "retry-request": "^1.3.0", "rgb-hex": "^1.0.0", "split-array-stream": "^1.0.0", "stream-events": "^1.0.1", @@ -138,6 +140,7 @@ "multiline": "^1.0.2", "node-uuid": "^1.4.3", "normalize-newline": "^2.0.0", + "sinon": "^1.17.4", "tmp": "0.0.27" }, "scripts": { diff --git a/scripts/docs.js b/scripts/docs.js index 49733eccfc5..faee12a0185 100644 --- a/scripts/docs.js +++ b/scripts/docs.js @@ -31,7 +31,8 @@ var IGNORE = [ './lib/datastore/entity.js', './lib/datastore/request.js', './lib/pubsub/iam.js', - './lib/storage/acl.js' + './lib/storage/acl.js', + './lib/bigtable/mutation.js' ]; function isPublic(block) { diff --git a/system-test/bigtable.js b/system-test/bigtable.js new file mode 100644 index 00000000000..76a776a748d --- /dev/null +++ b/system-test/bigtable.js @@ -0,0 +1,710 @@ +/*! + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var assert = require('assert'); +var async = require('async'); +var uuid = require('node-uuid'); +var exec = require('methmeth'); + +var Table = require('../lib/bigtable/table.js'); +var Family = require('../lib/bigtable/family.js'); +var Row = require('../lib/bigtable/row.js'); + +var env = require('./env.js'); +var gcloud = require('../lib/index.js')(env); + +var clusterName = process.env.GCLOUD_TESTS_BIGTABLE_CLUSTER; +var zoneName = process.env.GCLOUD_TESTS_BIGTABLE_ZONE; + +var isTestable = clusterName && zoneName; + +function generateTableName() { + return 'test-table-' + uuid.v4(); +} + +(isTestable ? describe : describe.skip)('Bigtable', function() { + var bigtable = gcloud.bigtable({ + cluster: clusterName, + zone: zoneName + }); + + var TABLE_NAME = generateTableName(); + var TABLE = bigtable.table(TABLE_NAME); + + before(function(done) { + bigtable.getTables(function(err, tables) { + if (err) { + done(err); + return; + } + + async.each(tables, exec('delete'), function(err) { + if (err) { + done(err); + return; + } + + TABLE.create(done); + }); + }); + }); + + after(function() { + TABLE.delete(); + }); + + describe('tables', function() { + + it('should retrieve a list of tables', function(done) { + bigtable.getTables(function(err, tables) { + assert.ifError(err); + assert(tables[0] instanceof Table); + done(); + }); + }); + + it('should check if a table exists', function(done) { + TABLE.exists(function(err, exists) { + assert.ifError(err); + assert.strictEqual(exists, true); + done(); + }); + }); + + it('should check if a table does not exist', function(done) { + var table = bigtable.table('should-not-exist'); + + table.exists(function(err, exists) { + assert.ifError(err); + assert.strictEqual(exists, false); + done(); + }); + }); + + it('should get a table', function(done) { + var table = bigtable.table(TABLE_NAME); + + table.get(function(err, table_) { + assert.ifError(err); + assert.strictEqual(table, table_); + done(); + }); + }); + + it('should delete a table', function(done) { + var table = bigtable.table(generateTableName()); + + async.series([ + table.create.bind(table), + table.delete.bind(table) + ], done); + }); + + it('should get the tables metadata', function(done) { + TABLE.getMetadata(function(err, metadata) { + assert.strictEqual(metadata.name, TABLE.id); + done(); + }); + }); + + }); + + describe('column families', function() { + var FAMILY_NAME = 'presidents'; + var FAMILY = TABLE.family(FAMILY_NAME); + + before(function(done) { + FAMILY.create(done); + }); + + it('should get a list of families', function(done) { + TABLE.getFamilies(function(err, families) { + assert.ifError(err); + assert.strictEqual(families.length, 1); + assert(families[0] instanceof Family); + assert.strictEqual(families[0].name, FAMILY.name); + done(); + }); + }); + + it('should get a family', function(done) { + var family = TABLE.family(FAMILY_NAME); + + family.get(function(err, family) { + assert.ifError(err); + assert(family instanceof Family); + assert.strictEqual(family.name, FAMILY.name); + done(); + }); + }); + + it('should check if a family exists', function(done) { + FAMILY.exists(function(err, exists) { + assert.ifError(err); + assert.strictEqual(exists, true); + done(); + }); + }); + + it('should check if a family does not exist', function(done) { + var family = TABLE.family('prezzies'); + + family.exists(function(err, exists) { + assert.ifError(err); + assert.strictEqual(exists, false); + done(); + }); + }); + + it('should get the column family metadata', function(done) { + FAMILY.getMetadata(function(err, metadata) { + assert.ifError(err); + assert.strictEqual(FAMILY.metadata, metadata); + done(); + }); + }); + + it('should update a column family', function(done) { + var rule = { + age: { + seconds: 10000, + nanos: 10000 + }, + union: true + }; + + FAMILY.setMetadata({ rule: rule }, function(err, metadata_) { + assert.ifError(err); + + var maxAge_ = metadata_.gcRule.maxAge; + + assert.equal(maxAge_.seconds, rule.age.seconds); + assert.strictEqual(maxAge_.nanas, rule.age.nanas); + done(); + }); + }); + + it('should delete a column family', function(done) { + FAMILY.delete(done); + }); + + }); + + describe('rows', function() { + + before(function(done) { + async.each(['follows', 'traits'], function(family, callback) { + TABLE.createFamily(family, callback); + }, done); + }); + + describe('inserting data', function() { + + it('should insert rows', function(done) { + var rows = [{ + key: 'gwashington', + data: { + follows: { + jadams: 1 + } + } + }, { + key: 'tjefferson', + data: { + follows: { + gwashington: 1, + jadams: 1 + } + } + }, { + key: 'jadams', + data: { + follows: { + gwashington: 1, + tjefferson: 1 + } + } + }]; + + TABLE.insert(rows, function(err) { + assert.ifError(err); + done(); + }); + }); + + it('should create an individual row', function(done) { + var row = TABLE.row('alincoln'); + var rowData = { + follows: { + gwashington: 1, + jadams: 1, + tjefferson: 1 + } + }; + + row.create(rowData, done); + }); + + it('should insert individual cells', function(done) { + var row = TABLE.row('gwashington'); + + var rowData = { + follows: { + jadams: 1 + } + }; + + row.save(rowData, done); + }); + + it('should allow for user specified timestamps', function(done) { + var row = TABLE.row('gwashington'); + + var rowData = { + follows: { + jadams: { + value: 1, + timestamp: new Date('March 22, 1986') + } + } + }; + + row.save(rowData, done); + }); + + it('should increment a column value', function(done) { + var row = TABLE.row('gwashington'); + var increment = 5; + + row.increment('follows:increment', increment, function(err, value) { + assert.ifError(err); + assert.strictEqual(value, increment); + done(); + }); + }); + + it('should apply read/modify/write rules to a row', function(done) { + var row = TABLE.row('gwashington'); + var rule = { + column: 'traits:teeth', + append: '-wood' + }; + + row.createRules(rule, function(err) { + assert.ifError(err); + + row.save('traits:teeth', 'shiny', function(err) { + assert.ifError(err); + + row.get(['traits:teeth'], function(err, data) { + assert.ifError(err); + assert(data.traits.teeth[0].value, 'shiny-wood'); + done(); + }); + }); + }); + }); + + it('should check and mutate a row', function(done) { + var row = TABLE.row('gwashington'); + var filter = { + family: 'follows', + value: 'alincoln' + }; + + var batch = [{ + method: 'delete', + data: ['follows:lincoln'] + }]; + + row.filter(filter, null, batch, function(err, matched) { + assert.ifError(err); + assert(matched); + done(); + }); + }); + + }); + + describe('fetching data', function() { + + it('should get rows', function(done) { + TABLE.getRows(function(err, rows) { + assert.ifError(err); + assert.strictEqual(rows.length, 4); + assert(rows[0] instanceof Row); + done(); + }); + }); + + it('should get rows via stream', function(done) { + var rows = []; + + TABLE.getRows() + .on('error', done) + .on('data', function(row) { + assert(row instanceof Row); + rows.push(row); + }) + .on('end', function() { + assert.strictEqual(rows.length, 4); + done(); + }); + }); + + it('should fetch an individual row', function(done) { + var row = TABLE.row('alincoln'); + + row.get(function(err, row_) { + assert.ifError(err); + assert.strictEqual(row, row_); + done(); + }); + }); + + it('should fetch individual cells of a row', function(done) { + var row = TABLE.row('alincoln'); + + row.get(['follows:gwashington'], function(err, data) { + assert.ifError(err); + assert.strictEqual(data.follows.gwashington[0].value, 1); + done(); + }); + }); + + it('should get sample row keys', function(done) { + TABLE.sampleRowKeys(function(err, keys) { + assert.ifError(err); + assert(keys.length > 0); + done(); + }); + }); + + it('should get sample row keys via stream', function(done) { + var keys = []; + + TABLE.sampleRowKeys() + .on('error', done) + .on('data', function(rowKey) { + keys.push(rowKey); + }) + .on('end', function() { + assert(keys.length > 0); + done(); + }); + }); + + describe('filters', function() { + + it('should get rows via column data', function(done) { + var filter = { + column: 'gwashington' + }; + + TABLE.getRows({ filter: filter }, function(err, rows) { + assert.ifError(err); + assert.strictEqual(rows.length, 3); + + var keys = rows.map(function(row) { + return row.id; + }).sort(); + + assert.deepEqual(keys, [ + 'alincoln', + 'jadams', + 'tjefferson' + ]); + + done(); + }); + }); + + it('should get rows that satisfy the cell limit', function(done) { + var entry = { + key: 'alincoln', + data: { + follows: { + tjefferson: 1 + } + } + }; + + var filter = [{ + row: 'alincoln' + }, { + column: { + name: 'tjefferson', + cellLimit: 1 + } + }]; + + TABLE.insert(entry, function(err) { + assert.ifError(err); + + TABLE.getRows({ filter: filter }, function(err, rows) { + assert.ifError(err); + var rowData = rows[0].data; + assert(rowData.follows.tjefferson.length, 1); + done(); + }); + }); + }); + + it('should get a range of columns', function(done) { + var filter = [{ + row: 'tjefferson' + }, { + column: { + family: 'follows', + start: 'gwashington', + end: 'jadams' + } + }]; + + TABLE.getRows({ filter: filter }, function(err, rows) { + assert.ifError(err); + + rows.forEach(function(row) { + var keys = Object.keys(row.data.follows).sort(); + + assert.deepEqual(keys, [ + 'gwashington', + 'jadams' + ]); + }); + + done(); + }); + }); + + it('should run a conditional filter', function(done) { + var filter = { + condition: { + test: [{ + row: 'gwashington' + }, { + family: 'follows' + }, { + column: 'tjefferson' + }], + pass: { + row: 'gwashington' + }, + fail: { + row: 'tjefferson' + } + } + }; + + TABLE.getRows({ filter: filter }, function(err, rows) { + assert.ifError(err); + assert.strictEqual(rows.length, 1); + assert.strictEqual(rows[0].id, 'tjefferson'); + done(); + }); + }); + + it('should only get cells for a specific family', function(done) { + var entries = [{ + key: 'gwashington', + data: { + traits: { + teeth: 'wood' + } + } + }]; + + var filter = { + family: 'traits' + }; + + TABLE.insert(entries, function(err) { + assert.ifError(err); + + TABLE.getRows({ filter: filter }, function(err, rows) { + assert.ifError(err); + assert(rows.length > 0); + + var families = Object.keys(rows[0].data); + assert.deepEqual(families, ['traits']); + done(); + }); + }); + }); + + it('should interleave filters', function(done) { + var filter = [{ + interleave: [ + [{ + row: 'gwashington' + }], [{ + row: 'tjefferson' + }] + ] + }]; + + TABLE.getRows({ filter: filter }, function(err, rows) { + assert.ifError(err); + assert.strictEqual(rows.length, 2); + + var ids = rows.map(function(row) { + return row.id; + }).sort(); + + assert.deepEqual(ids, [ + 'gwashington', + 'tjefferson' + ]); + + done(); + }); + }); + + it('should apply labels to the results', function(done) { + var filter = { + label: 'test-label' + }; + + TABLE.getRows({ filter: filter }, function(err, rows) { + assert.ifError(err); + + rows.forEach(function(row) { + var follows = row.data.follows; + + Object.keys(follows).forEach(function(column) { + follows[column].forEach(function(cell) { + assert.deepEqual(cell.labels, [filter.label]); + }); + }); + }); + + done(); + }); + }); + + it('should run a regex against the row id', function(done) { + var filter = { + row: /[a-z]+on$/ + }; + + TABLE.getRows({ filter: filter }, function(err, rows) { + assert.ifError(err); + + var keys = rows.map(function(row) { + return row.id; + }).sort(); + + assert.deepEqual(keys, [ + 'gwashington', + 'tjefferson' + ]); + + done(); + }); + }); + + it('should run a sink filter', function(done) { + var filter = [{ + row: 'alincoln' + }, { + family: 'follows' + }, { + interleave: [ + [{ + all: true + }], [{ + label: 'prezzy' + }, { + sink: true + }] + ] + }, { + column: 'gwashington' + }]; + + TABLE.getRows({ filter: filter }, function(err, rows) { + assert.ifError(err); + + var columns = Object.keys(rows[0].data.follows).sort(); + + assert.deepEqual(columns, [ + 'gwashington', + 'jadams', + 'tjefferson' + ]); + + done(); + }); + }); + + it('should accept a date range', function(done) { + var filter = { + time: { + start: new Date('March 21, 1986'), + end: new Date('March 23, 1986') + } + }; + + TABLE.getRows({ filter: filter }, function(err, rows) { + assert.ifError(err); + assert(rows.length > 0); + done(); + }); + }); + + }); + + }); + + describe('deleting rows', function() { + + it('should delete specific cells', function(done) { + var row = TABLE.row('alincoln'); + + row.deleteCells(['follows:gwashington'], done); + }); + + it('should delete a family', function(done) { + var row = TABLE.row('gwashington'); + + row.deleteCells(['traits'], done); + }); + + it('should delete all the cells', function(done) { + var row = TABLE.row('alincoln'); + + row.delete(done); + }); + + it('should delete all the rows', function(done) { + TABLE.deleteRows(function(err) { + assert.ifError(err); + + TABLE.getRows(function(err, rows) { + assert.ifError(err); + assert.strictEqual(rows.length, 0); + done(); + }); + }); + }); + + }); + + }); + +}); diff --git a/test/bigtable/family.js b/test/bigtable/family.js new file mode 100644 index 00000000000..ab5a58e6434 --- /dev/null +++ b/test/bigtable/family.js @@ -0,0 +1,329 @@ +/** + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var assert = require('assert'); +var mockery = require('mockery-next'); +var nodeutil = require('util'); +var util = require('../../lib/common/util.js'); +var format = require('string-format-obj'); + +var GrpcServiceObject = require('../../lib/common/grpc-service-object.js'); + +function FakeGrpcServiceObject() { + this.calledWith_ = arguments; + GrpcServiceObject.apply(this, arguments); +} + +nodeutil.inherits(FakeGrpcServiceObject, GrpcServiceObject); + +describe('Bigtable/Family', function() { + var FAMILY_NAME = 'family-test'; + var TABLE = { + id: 'my-table', + getFamilies: util.noop, + createFamily: util.noop + }; + + var FAMILY_ID = format('{t}/columnFamilies/{f}', { + t: TABLE.id, + f: FAMILY_NAME + }); + + var Family; + var family; + var FamilyError; + + before(function() { + mockery.registerMock( + '../../lib/common/grpc-service-object', FakeGrpcServiceObject); + + mockery.enable({ + useCleanCache: true, + warnOnUnregistered: false + }); + + Family = require('../../lib/bigtable/family.js'); + FamilyError = Family.FamilyError; + }); + + after(function() { + mockery.deregisterAll(); + mockery.disable(); + }); + + beforeEach(function() { + family = new Family(TABLE, FAMILY_NAME); + }); + + describe('instantiation', function() { + it('should inherit from GrpcServiceObject', function() { + var config = family.calledWith_[0]; + + assert(family instanceof FakeGrpcServiceObject); + assert.strictEqual(config.parent, TABLE); + assert.strictEqual(config.id, FAMILY_ID); + assert.deepEqual(config.methods, { + create: true, + exists: true, + get: true, + delete: { + protoOpts: { + service: 'BigtableTableService', + method: 'deleteColumnFamily' + }, + reqOpts: { + name: FAMILY_ID + } + } + }); + assert.strictEqual(typeof config.createMethod, 'function'); + }); + + it('should call Table#createFamily for the create method', function(done) { + var fakeOptions = {}; + + TABLE.createFamily = function(name, options, callback) { + assert.strictEqual(name, FAMILY_NAME); + assert.strictEqual(options, fakeOptions); + callback(null, family); // done + }; + + family.create(fakeOptions, done); + }); + }); + + describe('formatName_', function() { + it('should format the column family name', function() { + var formatted = Family.formatName_(TABLE.id, FAMILY_NAME); + + assert.strictEqual(formatted, FAMILY_ID); + }); + + it('should not re-format the name', function() { + var formatted = Family.formatName_(TABLE.id, FAMILY_ID); + + assert.strictEqual(formatted, FAMILY_ID); + }); + }); + + describe('formatRule_', function() { + it('should capture the max age option', function() { + var originalRule = { + age: 10 + }; + + var rule = Family.formatRule_(originalRule); + + assert.deepEqual(rule, { + maxAge: originalRule.age + }); + }); + + it('should capture the max number of versions option', function() { + var originalRule = { + versions: 10 + }; + + var rule = Family.formatRule_(originalRule); + + assert.deepEqual(rule, { + maxNumVersions: originalRule.versions + }); + }); + + it('should create a union rule', function() { + var originalRule = { + age: 10, + union: true + }; + + var rule = Family.formatRule_(originalRule); + + assert.deepEqual(rule, { + union: { + rules: [{ + maxAge: originalRule.age + }] + } + }); + }); + + it('should create an intersecting rule', function() { + var originalRule = { + versions: 2, + intersection: true + }; + + var rule = Family.formatRule_(originalRule); + + assert.deepEqual(rule, { + intersection: { + rules: [{ + maxNumVersions: originalRule.versions + }] + } + }); + }); + }); + + describe('getMetadata', function() { + it('should return an error to the callback', function(done) { + var err = new Error('err'); + var response = {}; + + family.parent.getFamilies = function(callback) { + callback(err, null, response); + }; + + family.getMetadata(function(err_, metadata, apiResponse) { + assert.strictEqual(err, err_); + assert.strictEqual(response, apiResponse); + done(); + }); + }); + + it('should update the metadata', function(done) { + var FAMILY = new Family(TABLE, FAMILY_NAME); + var response = { + families: {} + }; + + FAMILY.metadata = { + a: 'a', + b: 'b' + }; + + family.parent.getFamilies = function(callback) { + callback(null, [FAMILY], response); + }; + + family.getMetadata(function(err, metadata, apiResponse) { + assert.ifError(err); + assert.strictEqual(FAMILY.metadata, metadata); + assert.strictEqual(apiResponse, response); + done(); + }); + }); + + it('should throw a custom error', function(done) { + var response = {}; + + family.parent.getFamilies = function(callback) { + callback(null, [], response); + }; + + family.getMetadata(function(err, metadata, apiResponse) { + assert(err instanceof FamilyError); + assert.strictEqual(metadata, null); + assert.strictEqual(apiResponse, response); + done(); + }); + }); + }); + + describe('setMetadata', function() { + it('should provide the proper request options', function(done) { + family.request = function(protoOpts, reqOpts, callback) { + assert.deepEqual(protoOpts, { + service: 'BigtableTableService', + method: 'updateColumnFamily' + }); + + assert.strictEqual(reqOpts.name, FAMILY_ID); + callback(); + }; + + family.setMetadata({}, done); + }); + + it('should respect the gc expression option', function(done) { + var metadata = { + rule: 'a b c' + }; + + family.request = function(p, reqOpts) { + assert.strictEqual(reqOpts.gcExpression, metadata.rule); + done(); + }; + + family.setMetadata(metadata, assert.ifError); + }); + + it('should respect the gc rule option', function(done) { + var formatRule = Family.formatRule_; + + var formattedRule = { + a: 'a', + b: 'b' + }; + + var metadata = { + rule: { + c: 'c', + d: 'd' + } + }; + + Family.formatRule_ = function(rule) { + assert.strictEqual(rule, metadata.rule); + return formattedRule; + }; + + family.request = function(p, reqOpts) { + assert.strictEqual(reqOpts.gcRule, formattedRule); + Family.formatRule_ = formatRule; + done(); + }; + + family.setMetadata(metadata, assert.ifError); + }); + + it('should respect the updated name option', function(done) { + var formatName = Family.formatName_; + var fakeName = 'a/b/c'; + + var metadata = { + name: 'new-name' + }; + + Family.formatName_ = function(parent, newName) { + assert.strictEqual(parent, TABLE.id); + assert.strictEqual(newName, metadata.name); + return fakeName; + }; + + family.request = function(p, reqOpts) { + assert.strictEqual(reqOpts.name, fakeName); + Family.formatName_ = formatName; + done(); + }; + + family.setMetadata(metadata, assert.ifError); + }); + }); + + describe('FamilyError', function() { + it('should set the code and message', function() { + var err = new FamilyError(FAMILY_NAME); + + assert.strictEqual(err.code, 404); + assert.strictEqual(err.message, + 'Column family not found: ' + FAMILY_NAME + '.'); + }); + }); +}); diff --git a/test/bigtable/filter.js b/test/bigtable/filter.js new file mode 100644 index 00000000000..201aaf78e53 --- /dev/null +++ b/test/bigtable/filter.js @@ -0,0 +1,624 @@ +/** + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var assert = require('assert'); +var mockery = require('mockery-next'); +var sinon = require('sinon').sandbox.create(); + +var FakeMutation = { + convertToBytes: sinon.spy(function(value) { + return value; + }), + createTimeRange: sinon.stub() +}; + +describe('Bigtable/Filter', function() { + var Filter; + var filter; + + before(function() { + mockery.registerMock('../../lib/bigtable/mutation', FakeMutation); + + mockery.enable({ + useCleanCache: true, + warnOnUnregistered: false + }); + + Filter = require('../../lib/bigtable/filter'); + }); + + after(function() { + mockery.deregisterAll(); + mockery.disable(); + }); + + beforeEach(function() { + filter = new Filter(); + }); + + afterEach(function() { + sinon.restore(); + FakeMutation.convertToBytes.reset(); + }); + + describe('instantiation', function() { + it('should create an empty array of filters', function() { + assert.deepEqual(filter.filters_, []); + }); + }); + + describe('convertToRegExpString', function() { + it('should convert a RegExp to a string', function() { + var str = Filter.convertToRegExpString(/\d+/); + + assert.strictEqual(str, '\\d+'); + }); + + it('should convert an Array of strings to a single string', function() { + var things = ['a', 'b', 'c']; + var str = Filter.convertToRegExpString(things); + + assert.strictEqual(str, '(a|b|c)'); + }); + + it('should not do anything to a string', function() { + var str1 = 'hello'; + var str2 = Filter.convertToRegExpString(str1); + + assert.strictEqual(str1, str2); + }); + + it('should convert a number to a string', function() { + var str = Filter.convertToRegExpString(1); + + assert.strictEqual(str, '1'); + }); + + it('should throw an error for unknown types', function() { + var errorMessage = /Can\'t convert to RegExp String from unknown type\./; + + assert.throws(function() { + Filter.convertToRegExpString(true); + }, errorMessage); + }); + }); + + describe('createRange', function() { + it('should create a range object', function() { + var start = 'a'; + var end = 'b'; + var key = 'Key'; + + var range = Filter.createRange(start, end, key); + + assert.deepEqual(range, { + startKeyInclusive: start, + endKeyInclusive: end + }); + }); + + it('should only create start bound', function() { + var start = 'a'; + var key = 'Key'; + + var range = Filter.createRange(start, null, key); + + assert(FakeMutation.convertToBytes.calledWithExactly(start)); + assert.deepEqual(range, { + startKeyInclusive: start + }); + }); + + it('should only create an end bound', function() { + var end = 'b'; + var key = 'Key'; + + var range = Filter.createRange(null, end, key); + + assert(FakeMutation.convertToBytes.calledWithExactly(end)); + assert.deepEqual(range, { + endKeyInclusive: end + }); + }); + + it('should optionally accept inclusive flags', function() { + var start = { + value: 'a', + inclusive: false + }; + + var end = { + value: 'b', + inclusive: false + }; + + var key = 'Key'; + + var range = Filter.createRange(start, end, key); + + assert.deepEqual(range, { + startKeyExclusive: start.value, + endKeyExclusive: end.value + }); + }); + }); + + describe('parse', function() { + it('should call each individual filter method', function() { + sinon.spy(Filter.prototype, 'row'); + sinon.spy(Filter.prototype, 'value'); + + var fakeFilter = [{ + row: 'a' + }, { + value: 'b' + }]; + + Filter.parse(fakeFilter); + + assert.strictEqual(Filter.prototype.row.callCount, 1); + assert(Filter.prototype.row.calledWithExactly('a')); + + assert.strictEqual(Filter.prototype.value.callCount, 1); + assert(Filter.prototype.value.calledWithExactly('b')); + }); + + it('should throw an error for unknown filters', function() { + var fakeFilter = [{ + wat: 'a' + }]; + + assert.throws( + Filter.parse.bind(null, fakeFilter), + Filter.FilterError + ); + }); + + it('should return the filter in JSON form', function() { + var fakeProto = { a: 'a' }; + var fakeFilter = [{ + column: 'a' + }]; + + sinon.stub(Filter.prototype, 'toProto').returns(fakeProto); + + var parsedFilter = Filter.parse(fakeFilter); + + assert.strictEqual(parsedFilter, fakeProto); + assert.strictEqual(Filter.prototype.toProto.callCount, 1); + }); + }); + + describe('all', function() { + it('should create a pass all filter when set to true', function(done) { + filter.set = function(filterName, value) { + assert.strictEqual(filterName, 'passAllFilter'); + assert.strictEqual(value, true); + done(); + }; + + filter.all(true); + }); + + it('should create a block all filter when set to false', function(done) { + filter.set = function(filterName, value) { + assert.strictEqual(filterName, 'blockAllFilter'); + assert.strictEqual(value, true); + done(); + }; + + filter.all(false); + }); + }); + + describe('column', function() { + it('should set the column qualifier regex filter', function(done) { + var column = { + name: 'fake-column' + }; + + var spy = sinon.stub(Filter, 'convertToRegExpString', function(value) { + return value; + }); + + filter.set = function(filterName, value) { + assert.strictEqual(filterName, 'columnQualifierRegexFilter'); + assert.strictEqual(value, column.name); + assert(spy.calledWithExactly(column.name)); + assert(FakeMutation.convertToBytes.calledWithExactly(column.name)); + done(); + }; + + filter.column(column); + }); + + it('should accept the short-hand version of column', function(done) { + var column = 'fake-column'; + + filter.set = function(filterName, value) { + assert.strictEqual(filterName, 'columnQualifierRegexFilter'); + assert.strictEqual(value, column); + done(); + }; + + filter.column(column); + }); + + it('should accept the cells per column limit filter', function(done) { + var column = { + cellLimit: 10 + }; + + filter.set = function(filterName, value) { + assert.strictEqual(filterName, 'cellsPerColumnLimitFilter'); + assert.strictEqual(value, column.cellLimit); + done(); + }; + + filter.column(column); + }); + + it('should accept the column range filter', function(done) { + var fakeRange = { + a: 'a', + b: 'b' + }; + var column = { + start: 'a', + end: 'b' + }; + + var spy = sinon.stub(Filter, 'createRange').returns(fakeRange); + + filter.set = function(filterName, value) { + assert.strictEqual(filterName, 'columnRangeFilter'); + assert.strictEqual(value, fakeRange); + assert(spy.calledWithExactly(column.start, column.end, 'Qualifier')); + done(); + }; + + filter.column(column); + }); + }); + + describe('condition', function() { + it('should create a condition filter', function(done) { + var condition = { + test: { a: 'a' }, + pass: { b: 'b' }, + fail: { c: 'c' } + }; + + var spy = sinon.stub(Filter, 'parse', function(value) { + return value; + }); + + filter.set = function(filterName, value) { + assert.strictEqual(filterName, 'condition'); + assert.deepEqual(value, { + predicateFilter: condition.test, + trueFilter: condition.pass, + falseFilter: condition.fail + }); + + assert.strictEqual(spy.getCall(0).args[0], condition.test); + assert.strictEqual(spy.getCall(1).args[0], condition.pass); + assert.strictEqual(spy.getCall(2).args[0], condition.fail); + done(); + }; + + filter.condition(condition); + }); + }); + + describe('family', function() { + it('should create a family name regex filter', function(done) { + var familyName = 'fake-family'; + + var spy = sinon.stub(Filter, 'convertToRegExpString', function(value) { + return value; + }); + + filter.set = function(filterName, value) { + assert.strictEqual(filterName, 'familyNameRegexFilter'); + assert.strictEqual(value, familyName); + assert(spy.calledWithExactly(familyName)); + done(); + }; + + filter.family(familyName); + }); + }); + + describe('interleave', function() { + it('should create an interleave filter', function(done) { + var fakeFilters = [{}, {}, {}]; + + var spy = sinon.stub(Filter, 'parse', function(value) { + return value; + }); + + filter.set = function(filterName, value) { + assert.strictEqual(filterName, 'interleave'); + assert.strictEqual(value.filters[0], fakeFilters[0]); + assert.strictEqual(value.filters[1], fakeFilters[1]); + assert.strictEqual(value.filters[2], fakeFilters[2]); + assert.strictEqual(spy.getCall(0).args[0], fakeFilters[0]); + assert.strictEqual(spy.getCall(1).args[0], fakeFilters[1]); + assert.strictEqual(spy.getCall(2).args[0], fakeFilters[2]); + done(); + }; + + filter.interleave(fakeFilters); + }); + }); + + describe('label', function() { + it('should apply the label transformer', function(done) { + var label = 'label'; + + filter.set = function(filterName, value) { + assert.strictEqual(filterName, 'applyLabelTransformer'); + assert.strictEqual(value, label); + done(); + }; + + filter.label(label); + }); + }); + + describe('row', function() { + it('should apply the row key regex filter', function(done) { + var row = { + key: 'gwashinton' + }; + var convertedKey = 'abcd'; + + var spy = sinon.stub(Filter, 'convertToRegExpString', function() { + return convertedKey; + }); + + filter.set = function(filterName, value) { + assert.strictEqual(filterName, 'rowKeyRegexFilter'); + assert.strictEqual(value, convertedKey); + assert(spy.calledWithExactly(row.key)); + assert(FakeMutation.convertToBytes.calledWithExactly(convertedKey)); + done(); + }; + + filter.row(row); + }); + + it('should accept the short-hand version of row key', function(done) { + var rowKey = 'gwashington'; + + filter.set = function(filterName, value) { + assert.strictEqual(filterName, 'rowKeyRegexFilter'); + assert.strictEqual(value, rowKey); + done(); + }; + + filter.row(rowKey); + }); + + it('should set the row sample filter', function(done) { + var row = { + sample: 10 + }; + + filter.set = function(filterName, value) { + assert.strictEqual(filterName, 'rowSampleFilter'); + assert.strictEqual(value, row.sample); + done(); + }; + + filter.row(row); + }); + + it('should set the cells per row offset filter', function(done) { + var row = { + cellOffset: 10 + }; + + filter.set = function(filterName, value) { + assert.strictEqual(filterName, 'cellsPerRowOffsetFilter'); + assert.strictEqual(value, row.cellOffset); + done(); + }; + + filter.row(row); + }); + + it('should set the cells per row limit filter', function(done) { + var row = { + cellLimit: 10 + }; + + filter.set = function(filterName, value) { + assert.strictEqual(filterName, 'cellsPerRowLimitFilter'); + assert.strictEqual(value, row.cellLimit); + done(); + }; + + filter.row(row); + }); + }); + + describe('set', function() { + it('should create a filter object', function() { + var key = 'notARealFilter'; + var value = { a: 'b' }; + + filter.set(key, value); + + assert.strictEqual(filter.filters_[0][key], value); + }); + }); + + describe('sink', function() { + it('should set the sink filter', function(done) { + var sink = true; + + filter.set = function(filterName, value) { + assert.strictEqual(filterName, 'sink'); + assert.strictEqual(value, sink); + done(); + }; + + filter.sink(sink); + }); + }); + + describe('time', function() { + it('should set the timestamp range filter', function(done) { + var fakeTimeRange = { + start: 10, + end: 10 + }; + + var spy = FakeMutation.createTimeRange.returns(fakeTimeRange); + + filter.set = function(filterName, value) { + assert.strictEqual(filterName, 'timestampRangeFilter'); + assert.strictEqual(value, fakeTimeRange); + assert(spy.calledWithExactly(fakeTimeRange.start, fakeTimeRange.end)); + done(); + }; + + filter.time(fakeTimeRange); + }); + }); + + describe('toProto', function() { + it('should return a plain filter if there is only 1', function() { + filter.filters_ = [{}]; + + var filterProto = filter.toProto(); + + assert.strictEqual(filterProto, filter.filters_[0]); + }); + + it('should create a chain filter if there are multiple', function() { + filter.filters_ = [{}, {}]; + + var filterProto = filter.toProto(); + + assert.strictEqual(filterProto.chain.filters, filter.filters_); + }); + }); + + describe('value', function() { + it('should set the value regex filter', function(done) { + var value = { + value: 'fake-value' + }; + var fakeRegExValue = 'abcd'; + var fakeConvertedValue = 'dcba'; + + var regSpy = sinon.stub(Filter, 'convertToRegExpString', function() { + return fakeRegExValue; + }); + + var bytesSpy = FakeMutation.convertToBytes = sinon.spy(function() { + return fakeConvertedValue; + }); + + filter.set = function(filterName, val) { + assert.strictEqual(filterName, 'valueRegexFilter'); + assert.strictEqual(fakeConvertedValue, val); + assert(regSpy.calledWithExactly(value.value)); + assert(bytesSpy.calledWithExactly(fakeRegExValue)); + done(); + }; + + filter.value(value); + }); + + it('should accept the short-hand version of value', function(done) { + var value = 'fake-value'; + + var fakeRegExValue = 'abcd'; + var fakeConvertedValue = 'dcba'; + + var regSpy = sinon.stub(Filter, 'convertToRegExpString', function() { + return fakeRegExValue; + }); + + var bytesSpy = FakeMutation.convertToBytes = sinon.spy(function() { + return fakeConvertedValue; + }); + + filter.set = function(filterName, val) { + assert.strictEqual(filterName, 'valueRegexFilter'); + assert.strictEqual(fakeConvertedValue, val); + assert(regSpy.calledWithExactly(value)); + assert(bytesSpy.calledWithExactly(fakeRegExValue)); + done(); + }; + + filter.value(value); + }); + + it('should accept the value range filter', function(done) { + var fakeRange = { + a: 'a', + b: 'b' + }; + var value = { + start: 'a', + end: 'b' + }; + + var spy = sinon.stub(Filter, 'createRange', function() { + return fakeRange; + }); + + filter.set = function(filterName, val) { + assert.strictEqual(filterName, 'valueRangeFilter'); + assert.strictEqual(val, fakeRange); + assert(spy.calledWithExactly(value.start, value.end, 'Value')); + done(); + }; + + filter.value(value); + }); + + it('should apply the strip label transformer', function(done) { + var value = { + strip: true + }; + + filter.set = function(filterName, val) { + assert.strictEqual(filterName, 'stripValueTransformer'); + assert.strictEqual(val, value.strip); + done(); + }; + + filter.value(value); + }); + }); + + describe('FilterError', function() { + it('should set the correct message', function() { + var err = new Filter.FilterError('test'); + + assert.strictEqual(err.message, 'Unknown filter: test.'); + }); + }); + +}); diff --git a/test/bigtable/index.js b/test/bigtable/index.js new file mode 100644 index 00000000000..6d6698263a6 --- /dev/null +++ b/test/bigtable/index.js @@ -0,0 +1,325 @@ +/** + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var assert = require('assert'); +var mockery = require('mockery-next'); +var nodeutil = require('util'); +var extend = require('extend'); +var googleProtoFiles = require('google-proto-files'); +var format = require('string-format-obj'); +var sinon = require('sinon').sandbox.create(); + +var GrpcService = require('../../lib/common/grpc-service.js'); +var util = require('../../lib/common/util.js'); +var Table = require('../../lib/bigtable/table.js'); + +var fakeUtil = extend({}, util); + +function FakeGrpcService() { + this.calledWith_ = arguments; + GrpcService.apply(this, arguments); +} + +nodeutil.inherits(FakeGrpcService, GrpcService); + +function FakeTable() { + this.calledWith_ = arguments; + Table.apply(this, arguments); +} + +describe('Bigtable', function() { + var PROJECT_ID = 'test-project'; + var ZONE = 'test-zone'; + var CLUSTER = 'test-cluster'; + + var CLUSTER_NAME = format('projects/{p}/zones/{z}/clusters/{c}', { + p: PROJECT_ID, + z: ZONE, + c: CLUSTER + }); + + var Bigtable; + var bigtable; + + before(function() { + mockery.registerMock('../../lib/common/grpc-service.js', FakeGrpcService); + mockery.registerMock('../../lib/common/util.js', fakeUtil); + mockery.registerMock('../../lib/bigtable/table.js', FakeTable); + + mockery.enable({ + useCleanCache: true, + warnOnUnregistered: false + }); + + Bigtable = require('../../lib/bigtable/index.js'); + }); + + after(function() { + mockery.deregisterAll(); + mockery.disable(); + }); + + afterEach(function() { + sinon.restore(); + }); + + beforeEach(function() { + bigtable = new Bigtable({ + projectId: PROJECT_ID, + zone: ZONE, + cluster: CLUSTER + }); + }); + + describe('instantiation', function() { + it('should normalize the arguments', function() { + var normalizeArguments = fakeUtil.normalizeArguments; + var normalizeArgumentsCalled = false; + var fakeOptions = { + projectId: PROJECT_ID, + zone: ZONE, + cluster: CLUSTER + }; + var fakeContext = {}; + + fakeUtil.normalizeArguments = function(context, options) { + normalizeArgumentsCalled = true; + assert.strictEqual(context, fakeContext); + assert.strictEqual(options, fakeOptions); + return options; + }; + + Bigtable.call(fakeContext, fakeOptions); + assert(normalizeArgumentsCalled); + + fakeUtil.normalizeArguments = normalizeArguments; + }); + + it('should localize the cluster name', function() { + assert.strictEqual(bigtable.clusterName, CLUSTER_NAME); + }); + + it('should inherit from GrpcService', function() { + assert(bigtable instanceof GrpcService); + + var calledWith = bigtable.calledWith_[0]; + + assert.strictEqual(calledWith.baseUrl, 'bigtable.googleapis.com'); + assert.strictEqual(calledWith.service, 'bigtable'); + assert.strictEqual(calledWith.apiVersion, 'v1'); + + assert.deepEqual(calledWith.protoServices, { + BigtableService: googleProtoFiles.bigtable.v1, + BigtableTableService: { + path: googleProtoFiles.bigtable.admin, + service: 'bigtable.admin.table' + } + }); + + assert.deepEqual(calledWith.scopes, [ + 'https://www.googleapis.com/auth/bigtable.admin', + 'https://www.googleapis.com/auth/bigtable.data' + ]); + }); + }); + + describe('formatTableName_', function() { + it('should return the last section of a formatted table name', function() { + var fakeTableName = 'projects/p/zones/z/clusters/c/tables/my-table'; + var formatted = Bigtable.formatTableName_(fakeTableName); + + assert.strictEqual(formatted, 'my-table'); + }); + + it('should do nothing if the table is name is not formatted', function() { + var fakeTableName = 'my-table'; + var formatted = Bigtable.formatTableName_(fakeTableName); + + assert.strictEqual(formatted, fakeTableName); + }); + }); + + describe('createTable', function() { + var TABLE_ID = 'my-table'; + + it('should provide the proper request options', function(done) { + bigtable.request = function(protoOpts, reqOpts) { + assert.deepEqual(protoOpts, { + service: 'BigtableTableService', + method: 'createTable' + }); + + assert.strictEqual(reqOpts.name, CLUSTER_NAME); + assert.strictEqual(reqOpts.tableId, TABLE_ID); + assert.deepEqual(reqOpts.table, { + granularity: 0 + }); + done(); + }; + + bigtable.createTable(TABLE_ID, assert.ifError); + }); + + it('should set the current operation', function(done) { + var options = { + operation: 'abc' + }; + + bigtable.request = function(protoOpts, reqOpts) { + assert.strictEqual(reqOpts.table.currentOperation, options.operation); + done(); + }; + + bigtable.createTable(TABLE_ID, options, assert.ifError); + }); + + it('should set the initial split keys', function(done) { + var options = { + splits: ['a', 'b'] + }; + + bigtable.request = function(protoOpts, reqOpts) { + assert.strictEqual(reqOpts.initialSplitKeys, options.splits); + done(); + }; + + bigtable.createTable(TABLE_ID, options, assert.ifError); + }); + + it('should return an error to the callback', function(done) { + var err = new Error('err'); + var response = {}; + + bigtable.request = function(protoOpts, reqOpts, callback) { + callback(err, response); + }; + + bigtable.createTable(TABLE_ID, function(err_, table, apiResponse) { + assert.strictEqual(err, err_); + assert.strictEqual(table, null); + assert.strictEqual(apiResponse, response); + done(); + }); + }); + + it('should return a Table object', function(done) { + var response = { + name: TABLE_ID + }; + + var fakeTable = {}; + + var tableSpy = sinon.stub(bigtable, 'table', function() { + return fakeTable; + }); + + bigtable.request = function(p, r, callback) { + callback(null, response); + }; + + bigtable.createTable(TABLE_ID, function(err, table, apiResponse) { + assert.ifError(err); + assert.strictEqual(table, fakeTable); + assert(tableSpy.calledWithExactly(response.name)); + assert.strictEqual(table.metadata, response); + assert.strictEqual(response, apiResponse); + done(); + }); + }); + }); + + describe('getTables', function() { + it('should provide the proper request options', function(done) { + bigtable.request = function(protoOpts, reqOpts) { + assert.deepEqual(protoOpts, { + service: 'BigtableTableService', + method: 'listTables' + }); + assert.strictEqual(reqOpts.name, CLUSTER_NAME); + done(); + }; + + bigtable.getTables(assert.ifError); + }); + + it('should return an error to the callback', function(done) { + var err = new Error('err'); + var response = {}; + + bigtable.request = function(p, r, callback) { + callback(err, response); + }; + + bigtable.getTables(function(err_, tables, apiResponse) { + assert.strictEqual(err, err_); + assert.strictEqual(tables, null); + assert.strictEqual(apiResponse, response); + done(); + }); + }); + + it('should return a list of Table objects', function(done) { + var fakeFormattedName = 'abcd'; + var response = { + tables: [{ + name: 'projects/p/zones/z/clusters/c/tables/my-table' + }] + }; + var fakeTable = {}; + + bigtable.request = function(p, r, callback) { + callback(null, response); + }; + + var tableSpy = sinon.stub(bigtable, 'table', function() { + return fakeTable; + }); + + var formatSpy = sinon.stub(Bigtable, 'formatTableName_', function() { + return fakeFormattedName; + }); + + bigtable.getTables(function(err, tables, apiResponse) { + assert.ifError(err); + + var table = tables[0]; + + assert.strictEqual(table, fakeTable); + assert(formatSpy.calledWithExactly(response.tables[0].name)); + assert(tableSpy.calledWithExactly(fakeFormattedName)); + assert.strictEqual(table.metadata, response.tables[0]); + assert.strictEqual(response, apiResponse); + done(); + }); + }); + }); + + describe('table', function() { + var TABLE_ID = 'table-id'; + + it('should return a table instance', function() { + var table = bigtable.table(TABLE_ID); + var args = table.calledWith_; + + assert(table instanceof FakeTable); + assert.strictEqual(args[0], bigtable); + assert.strictEqual(args[1], TABLE_ID); + }); + }); + +}); diff --git a/test/bigtable/mutation.js b/test/bigtable/mutation.js new file mode 100644 index 00000000000..4d2a8f8b63c --- /dev/null +++ b/test/bigtable/mutation.js @@ -0,0 +1,412 @@ +/** + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var assert = require('assert'); +var Mutation = require('../../lib/bigtable/mutation.js'); +var Int64 = require('node-int64'); +var sinon = require('sinon').sandbox.create(); + +describe('Bigtable/Mutation', function() { + + afterEach(function() { + sinon.restore(); + }); + + describe('instantiation', function() { + it('should localize all the mutation properties', function() { + var fakeData = { + key: 'a', + method: 'b', + data: 'c' + }; + + var mutation = new Mutation(fakeData); + + assert.strictEqual(mutation.key, fakeData.key); + assert.strictEqual(mutation.method, fakeData.method); + assert.strictEqual(mutation.data, fakeData.data); + }); + }); + + describe('convertFromBytes', function() { + it('should convert a base64 encoded number', function() { + var num = 10; + var encoded = new Int64(num).toBuffer().toString('base64'); + var decoded = Mutation.convertFromBytes(encoded); + + assert.strictEqual(num, decoded); + }); + + it('should convert a base64 encoded string', function() { + var message = 'Hello!'; + var encoded = new Buffer(message).toString('base64'); + var decoded = Mutation.convertFromBytes(encoded); + + assert.strictEqual(message, decoded); + }); + }); + + describe('convertToBytes', function() { + it('should pack numbers into int64 values', function() { + var num = 10; + var encoded = Mutation.convertToBytes(num); + var decoded = new Int64(encoded).toNumber(); + + assert.strictEqual(num, decoded); + }); + + it('should wrap the value in a buffer', function() { + var message = 'Hello!'; + var encoded = Mutation.convertToBytes(message); + + assert(encoded instanceof Buffer); + assert.strictEqual(encoded.toString(), message); + }); + + it('should simply return the value if it cannot wrap it', function() { + var message = true; + var notEncoded = Mutation.convertToBytes(message); + + assert(!(notEncoded instanceof Buffer)); + assert.strictEqual(message, notEncoded); + }); + }); + + describe('createTimeRange', function() { + it('should create a time range', function() { + var timestamp = Date.now(); + var dateObj = new Date(timestamp); + var range = Mutation.createTimeRange(dateObj, dateObj); + + assert.strictEqual(range.startTimestampMicros, timestamp); + assert.strictEqual(range.endTimestampMicros, timestamp); + }); + }); + + describe('encodeSetCell', function() { + var convert; + var convertCalls = []; + + before(function() { + convert = Mutation.convertToBytes; + Mutation.convertToBytes = function(value) { + convertCalls.push(value); + return value; + }; + }); + + after(function() { + Mutation.convertToBytes = convert; + }); + + beforeEach(function() { + convertCalls = []; + }); + + it('should encode a setCell mutation', function() { + var fakeMutation = { + follows: { + gwashington: 1, + alincoln: 1 + } + }; + + var cells = Mutation.encodeSetCell(fakeMutation); + + assert.strictEqual(cells.length, 2); + + assert.deepEqual(cells, [{ + setCell: { + familyName: 'follows', + columnQualifier: 'gwashington', + timestampMicros: -1, + value: 1 + } + }, { + setCell: { + familyName: 'follows', + columnQualifier: 'alincoln', + timestampMicros: -1, + value: 1 + } + }]); + + assert.strictEqual(convertCalls.length, 4); + assert.deepEqual(convertCalls, ['gwashington', 1, 'alincoln', 1]); + }); + + it('should optionally accept a timestamp', function() { + var timestamp = Date.now(); + var fakeMutation = { + follows: { + gwashington: { + value: 1, + timestamp: new Date(timestamp) + } + } + }; + + var cells = Mutation.encodeSetCell(fakeMutation); + + assert.deepEqual(cells, [{ + setCell: { + familyName: 'follows', + columnQualifier: 'gwashington', + timestampMicros: timestamp, + value: 1 + } + }]); + + assert.strictEqual(convertCalls.length, 2); + assert.deepEqual(convertCalls, ['gwashington', 1]); + }); + }); + + describe('encodeDelete', function() { + var convert; + var convertCalls = []; + + before(function() { + convert = Mutation.convertToBytes; + Mutation.convertToBytes = function(value) { + convertCalls.push(value); + return value; + }; + }); + + after(function() { + Mutation.convertToBytes = convert; + }); + + beforeEach(function() { + convertCalls = []; + }); + + it('should create a delete row mutation', function() { + var mutation = Mutation.encodeDelete(); + + assert.deepEqual(mutation, [{ + deleteFromRow: {} + }]); + }); + + it('should array-ify the input', function() { + var fakeKey = 'follows'; + var mutation = Mutation.encodeDelete(fakeKey); + + assert.deepEqual(mutation, [{ + deleteFromFamily: { + familyName: fakeKey + } + }]); + }); + + it('should create a delete family mutation', function() { + var fakeColumnName = { + family: 'followed', + qualifier: null + }; + + sinon.stub(Mutation, 'parseColumnName', function() { + return fakeColumnName; + }); + + var mutation = Mutation.encodeDelete(['follows']); + + assert.deepEqual(mutation, [{ + deleteFromFamily: { + familyName: fakeColumnName.family + } + }]); + }); + + it('should create a delete column mutation', function() { + var mutation = Mutation.encodeDelete(['follows:gwashington']); + + assert.deepEqual(mutation, [{ + deleteFromColumn: { + familyName: 'follows', + columnQualifier: 'gwashington', + timeRange: undefined + } + }]); + + assert.strictEqual(convertCalls.length, 1); + assert.strictEqual(convertCalls[0], 'gwashington'); + }); + + it('should optionally accept a timerange for column requests', function() { + var createTimeRange = Mutation.createTimeRange; + var timeCalls = []; + var fakeTimeRange = { a: 'a' }; + + var fakeMutationData = { + column: 'follows:gwashington', + time: { + start: 1, + end: 2 + } + }; + + Mutation.createTimeRange = function(start, end) { + timeCalls.push({ + start: start, + end: end + }); + return fakeTimeRange; + }; + + var mutation = Mutation.encodeDelete(fakeMutationData); + + assert.deepEqual(mutation, [{ + deleteFromColumn: { + familyName: 'follows', + columnQualifier: 'gwashington', + timeRange: fakeTimeRange + } + }]); + + assert.strictEqual(timeCalls.length, 1); + assert.deepEqual(timeCalls[0], fakeMutationData.time); + + Mutation.createTimeRange = createTimeRange; + }); + }); + + describe('parse', function() { + var toProto; + var toProtoCalled = false; + var fakeData = { a: 'a' }; + + before(function() { + toProto = Mutation.prototype.toProto; + Mutation.prototype.toProto = function() { + toProtoCalled = true; + return fakeData; + }; + }); + + after(function() { + Mutation.prototype.toProto = toProto; + }); + + it('should create a new mutation object and parse it', function() { + var fakeMutationData = { + key: 'a', + method: 'b', + data: 'c' + }; + + var mutation = Mutation.parse(fakeMutationData); + + assert.strictEqual(toProtoCalled, true); + assert.strictEqual(mutation, fakeData); + }); + + it('should parse a pre-existing mutation object', function() { + var data = new Mutation({ + key: 'a', + method: 'b', + data: [] + }); + + var mutation = Mutation.parse(data); + + assert.strictEqual(toProtoCalled, true); + assert.strictEqual(mutation, fakeData); + }); + }); + + describe('parseColumnName', function() { + it('should parse a column name', function() { + var parsed = Mutation.parseColumnName('a:b'); + + assert.strictEqual(parsed.family, 'a'); + assert.strictEqual(parsed.qualifier, 'b'); + }); + + it('should parse a family name', function() { + var parsed = Mutation.parseColumnName('a'); + + assert.strictEqual(parsed.family, 'a'); + assert.strictEqual(parsed.qualifier, undefined); + }); + }); + + describe('toProto', function() { + var convert; + var convertCalls = []; + + before(function() { + convert = Mutation.convertToBytes; + Mutation.convertToBytes = function(value) { + convertCalls.push(value); + return value; + }; + }); + + after(function() { + Mutation.convertToBytes = convert; + }); + + beforeEach(function() { + convertCalls = []; + }); + + it('should encode set cell mutations when method is insert', function() { + var fakeEncoded = [{ a: 'a' }]; + var data = { + key: 'a', + method: 'insert', + data: [] + }; + + Mutation.encodeSetCell = function(_data) { + assert.strictEqual(_data, data.data); + return fakeEncoded; + }; + + var mutation = new Mutation(data).toProto(); + + assert.strictEqual(mutation.mutations, fakeEncoded); + assert.strictEqual(mutation.rowKey, data.key); + assert.strictEqual(convertCalls[0], data.key); + }); + + it('should encode delete mutations when method is delete', function() { + var fakeEncoded = [{ b: 'b' }]; + var data = { + key: 'b', + method: 'delete', + data: [] + }; + + Mutation.encodeDelete = function(_data) { + assert.strictEqual(_data, data.data); + return fakeEncoded; + }; + + var mutation = new Mutation(data).toProto(); + + assert.strictEqual(mutation.mutations, fakeEncoded); + assert.strictEqual(mutation.rowKey, data.key); + assert.strictEqual(convertCalls[0], data.key); + }); + }); + +}); diff --git a/test/bigtable/row.js b/test/bigtable/row.js new file mode 100644 index 00000000000..fbfc7711f87 --- /dev/null +++ b/test/bigtable/row.js @@ -0,0 +1,801 @@ +/** + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var assert = require('assert'); +var mockery = require('mockery-next'); +var nodeutil = require('util'); +var sinon = require('sinon').sandbox.create(); + +var GrpcServiceObject = require('../../lib/common/grpc-service-object.js'); +var Mutation = require('../../lib/bigtable/mutation.js'); + +function FakeGrpcServiceObject() { + this.calledWith_ = arguments; + GrpcServiceObject.apply(this, arguments); +} + +nodeutil.inherits(FakeGrpcServiceObject, GrpcServiceObject); + +var ROW_ID = 'my-row'; +var CONVERTED_ROW_ID = 'my-converted-row'; +var TABLE = { + id: 'my-table' +}; + +var FakeMutation = { + methods: Mutation.methods, + convertToBytes: sinon.spy(function(value) { + if (value === ROW_ID) { + return CONVERTED_ROW_ID; + } + return value; + }), + convertFromBytes: sinon.spy(function(value) { + return value; + }), + parseColumnName: sinon.spy(function(column) { + return Mutation.parseColumnName(column); + }), + parse: sinon.spy(function(entry) { + return { + mutations: entry + }; + }) +}; + +var FakeFilter = { + parse: sinon.spy(function(filter) { + return filter; + }) +}; + +describe('Bigtable/Row', function() { + var Row; + var row; + + before(function() { + mockery.registerMock( + '../../lib/common/grpc-service-object', FakeGrpcServiceObject); + mockery.registerMock('../../lib/bigtable/mutation', FakeMutation); + mockery.registerMock('../../lib/bigtable/filter', FakeFilter); + + mockery.enable({ + useCleanCache: true, + warnOnUnregistered: false + }); + + Row = require('../../lib/bigtable/row.js'); + }); + + after(function() { + mockery.deregisterAll(); + mockery.disable(); + }); + + beforeEach(function() { + row = new Row(TABLE, ROW_ID); + }); + + afterEach(function() { + sinon.restore(); + + Object.keys(FakeMutation).forEach(function(spy) { + if (FakeMutation[spy].reset) { + FakeMutation[spy].reset(); + } + }); + }); + + describe('instantiation', function() { + it('should inherit from GrpcServiceObject', function() { + var config = row.calledWith_[0]; + + assert(row instanceof FakeGrpcServiceObject); + assert.strictEqual(config.parent, TABLE); + assert.deepEqual(config.methods, { + exists: true + }); + assert.strictEqual(config.id, ROW_ID); + }); + + it('should create an empty data object', function() { + assert.deepEqual(row.data, {}); + }); + }); + + describe('formatChunks_', function() { + var formatFamiliesSpy; + + beforeEach(function() { + formatFamiliesSpy = sinon.stub(Row, 'formatFamilies_'); + }); + + it('should not include chunks without commitRow', function() { + var chunks = [{ + rowConents: {} + }]; + var fakeFamilies = []; + + formatFamiliesSpy.returns(fakeFamilies); + + var formatted = Row.formatChunks_(chunks); + + assert.strictEqual(formatted, fakeFamilies); + assert.strictEqual(formatFamiliesSpy.callCount, 1); + assert.deepEqual(formatFamiliesSpy.getCall(0).args[0], []); + }); + + it('should ignore any chunks previous to a resetRow', function() { + var badData = {}; + var goodData = {}; + var fakeFamilies = [goodData, {}]; + + var chunks = [{ + rowContents: badData, + }, { + resetRow: true + }, { + rowContents: goodData + }, { + commitRow: true + }]; + + formatFamiliesSpy.returns(fakeFamilies); + + var formatted = Row.formatChunks_(chunks); + + assert.strictEqual(formatted, fakeFamilies); + assert.strictEqual(formatted.indexOf(badData), -1); + assert.strictEqual(formatFamiliesSpy.callCount, 1); + assert.deepEqual(formatFamiliesSpy.getCall(0).args[0], [goodData]); + }); + }); + + describe('formatFamilies_', function() { + var timestamp = Date.now(); + + var families = [{ + name: 'test-family', + columns: [{ + qualifier: 'test-column', + cells: [{ + value: 'test-value', + timestampMicros: timestamp, + labels: [] + }] + }] + }]; + + var formattedRowData = { + 'test-family': { + 'test-column': [{ + value: 'test-value', + timestamp: timestamp, + labels: [] + }] + } + }; + + it('should format the families into a user-friendly format', function() { + var formatted = Row.formatFamilies_(families); + assert.deepEqual(formatted, formattedRowData); + + var convertStpy = FakeMutation.convertFromBytes; + assert.strictEqual(convertStpy.callCount, 2); + assert.strictEqual(convertStpy.getCall(0).args[0], 'test-column'); + assert.strictEqual(convertStpy.getCall(1).args[0], 'test-value'); + }); + }); + + describe('create', function() { + it('should provide the proper request options', function(done) { + row.parent.mutate = function(entry) { + assert.strictEqual(entry.key, row.id); + assert.deepEqual(entry.data, {}); + assert.strictEqual(entry.method, Mutation.methods.INSERT); + done(); + }; + + row.create(assert.ifError); + }); + + it('should accept data to populate the row', function(done) { + var data = { + a: 'a', + b: 'b' + }; + + row.parent.mutate = function(entry) { + assert.strictEqual(entry.data, data); + done(); + }; + + row.create(data, assert.ifError); + }); + + it('should return an error to the callback', function(done) { + var err = new Error('err'); + var response = {}; + + row.parent.mutate = function(entry, callback) { + callback(err, response); + }; + + row.create(function(err_, row, apiResponse) { + assert.strictEqual(err, err_); + assert.strictEqual(row, null); + assert.strictEqual(response, apiResponse); + done(); + }); + }); + + it('should return the Row instance', function(done) { + var response = {}; + + row.parent.mutate = function(entry, callback) { + callback(null, response); + }; + + row.create(function(err, row_, apiResponse) { + assert.ifError(err); + assert.strictEqual(row, row_); + assert.strictEqual(response, apiResponse); + done(); + }); + }); + }); + + describe('createRules', function() { + var rules = [{ + column: 'a:b', + append: 'c', + increment: 1 + }]; + + it('should read/modify/write rules', function(done) { + row.request = function(grpcOpts, reqOpts, callback) { + assert.deepEqual(grpcOpts, { + service: 'BigtableService', + method: 'readModifyWriteRow' + }); + + assert.strictEqual(reqOpts.tableName, TABLE.id); + assert.strictEqual(reqOpts.rowKey, CONVERTED_ROW_ID); + + assert.deepEqual(reqOpts.rules, [{ + familyName: 'a', + columnQualifier: 'b', + appendValue: 'c', + incrementAmount: 1 + }]); + + var spy = FakeMutation.convertToBytes; + + assert.strictEqual(spy.getCall(0).args[0], 'b'); + assert.strictEqual(spy.getCall(1).args[0], 'c'); + assert.strictEqual(spy.getCall(2).args[0], ROW_ID); + callback(); + }; + + row.createRules(rules, done); + }); + + it('should return an error to the callback', function(done) { + var err = new Error('err'); + var response = {}; + + row.request = function(g, r, callback) { + callback(err, response); + }; + + row.createRules(rules, function(err_, apiResponse) { + assert.strictEqual(err, err_); + assert.strictEqual(response, apiResponse); + done(); + }); + }); + }); + + describe('filter', function() { + var mutations = [{ + method: 'insert', + data: { + a: 'a' + } + }]; + + var fakeMutations = { + mutations: [{ + a: 'b' + }] + }; + + beforeEach(function() { + FakeMutation.parse.reset(); + FakeFilter.parse.reset(); + }); + + it('should provide the proper request options', function(done) { + var filter = { + column: 'a' + }; + var fakeParsedFilter = { + column: 'b' + }; + + FakeFilter.parse = sinon.spy(function() { + return fakeParsedFilter; + }); + + FakeMutation.parse = sinon.spy(function() { + return fakeMutations; + }); + + row.request = function(grpcOpts, reqOpts) { + assert.deepEqual(grpcOpts, { + service: 'BigtableService', + method: 'checkAndMutateRow' + }); + + assert.strictEqual(reqOpts.tableName, TABLE.id); + assert.strictEqual(reqOpts.rowKey, CONVERTED_ROW_ID); + assert.deepEqual(reqOpts.predicateFilter, fakeParsedFilter); + assert.deepEqual(reqOpts.trueMutations, fakeMutations.mutations); + assert.deepEqual(reqOpts.falseMutations, fakeMutations.mutations); + + assert.strictEqual(FakeMutation.parse.callCount, 2); + assert.strictEqual(FakeMutation.parse.getCall(0).args[0], mutations[0]); + assert.strictEqual(FakeMutation.parse.getCall(1).args[0], mutations[0]); + + assert.strictEqual(FakeFilter.parse.callCount, 1); + assert(FakeFilter.parse.calledWithExactly(filter)); + done(); + }; + + row.filter(filter, mutations, mutations, assert.ifError); + }); + + it('should optionally accept onNoMatch mutations', function(done) { + row.request = function(g, reqOpts) { + assert.deepEqual(reqOpts.falseMutations, []); + assert.strictEqual(FakeMutation.parse.callCount, 1); + assert(FakeMutation.parse.calledWithExactly(mutations[0])); + done(); + }; + + row.filter({}, mutations, assert.ifError); + }); + + it('should optionally accept onMatch mutations', function(done) { + row.request = function(g, reqOpts) { + assert.deepEqual(reqOpts.trueMutations, []); + assert.strictEqual(FakeMutation.parse.callCount, 1); + assert(FakeMutation.parse.calledWithExactly(mutations[0])); + done(); + }; + + row.filter({}, null, mutations, assert.ifError); + }); + + it('should return an error to the callback', function(done) { + var err = new Error('err'); + var response = {}; + + row.request = function(g, r, callback) { + callback(err, response); + }; + + row.filter({}, mutations, function(err_, matched, apiResponse) { + assert.strictEqual(err, err_); + assert.strictEqual(matched, null); + assert.strictEqual(response, apiResponse); + done(); + }); + }); + + it('should return a matched flag', function(done) { + var response = { + predicateMatched: true + }; + + row.request = function(g, r, callback) { + callback(null, response); + }; + + row.filter({}, mutations, function(err, matched, apiResponse) { + assert.ifError(err); + assert(matched); + assert.strictEqual(response, apiResponse); + done(); + }); + }); + }); + + describe('delete', function() { + it('should provide the proper request options', function(done) { + row.parent.mutate = function(mutation, callback) { + assert.strictEqual(mutation.key, ROW_ID); + assert.strictEqual(mutation.method, FakeMutation.methods.DELETE); + callback(); + }; + + row.delete(done); + }); + }); + + describe('deleteCells', function() { + var columns = [ + 'a:b', + 'c' + ]; + + it('should provide the proper request options', function(done) { + row.parent.mutate = function(mutation, callback) { + assert.strictEqual(mutation.key, ROW_ID); + assert.strictEqual(mutation.data, columns); + assert.strictEqual(mutation.method, FakeMutation.methods.DELETE); + callback(); + }; + + row.deleteCells(columns, done); + }); + }); + + describe('get', function() { + it('should provide the proper request options', function(done) { + row.parent.getRows = function(reqOpts) { + assert.strictEqual(reqOpts.key, ROW_ID); + assert.strictEqual(reqOpts.filter, undefined); + assert.strictEqual(FakeMutation.parseColumnName.callCount, 0); + done(); + }; + + row.get(assert.ifError); + }); + + it('should create a filter for a single column', function(done) { + var keys = [ + 'a:b' + ]; + + var expectedFilter = [{ + family: 'a' + }, { + column: 'b' + }]; + + row.parent.getRows = function(reqOpts) { + assert.deepEqual(reqOpts.filter, expectedFilter); + assert.strictEqual(FakeMutation.parseColumnName.callCount, 1); + assert(FakeMutation.parseColumnName.calledWith(keys[0])); + done(); + }; + + row.get(keys, assert.ifError); + }); + + it('should create a filter for multiple columns', function(done) { + var keys = [ + 'a:b', + 'c:d' + ]; + + var expectedFilter = [{ + interleave: [ + [{ + family: 'a' + }, { + column: 'b' + }], [{ + family: 'c' + }, { + column: 'd' + }] + ] + }]; + + row.parent.getRows = function(reqOpts) { + assert.deepEqual(reqOpts.filter, expectedFilter); + + var spy = FakeMutation.parseColumnName; + + assert.strictEqual(spy.callCount, 2); + assert.strictEqual(spy.getCall(0).args[0], keys[0]); + assert.strictEqual(spy.getCall(1).args[0], keys[1]); + done(); + }; + + row.get(keys, assert.ifError); + }); + + it('should respect supplying only family names', function(done) { + var keys = [ + 'a' + ]; + + var expectedFilter = [{ + family: 'a' + }]; + + row.parent.getRows = function(reqOpts) { + assert.deepEqual(reqOpts.filter, expectedFilter); + assert.strictEqual(FakeMutation.parseColumnName.callCount, 1); + assert(FakeMutation.parseColumnName.calledWith(keys[0])); + done(); + }; + + row.get(keys, assert.ifError); + }); + + it('should return an error to the callback', function(done) { + var error = new Error('err'); + var response = {}; + + row.parent.getRows = function(r, callback) { + callback(error, null, response); + }; + + row.get(function(err, row_, apiResponse) { + assert.strictEqual(error, err); + assert.strictEqual(row_, null); + assert.strictEqual(response, apiResponse); + done(); + }); + }); + + it('should return a custom error if the row is not found', function(done) { + var response = {}; + + row.parent.getRows = function(r, callback) { + callback(null, [], response); + }; + + row.get(function(err, row_, apiResponse) { + assert(err instanceof Row.RowError); + assert.strictEqual(err.message, 'Unknown row: ' + row.id + '.'); + assert.deepEqual(row_, null); + assert.strictEqual(response, apiResponse); + done(); + }); + }); + + it('should update the row data upon success', function(done) { + var response = {}; + var fakeRow = new Row(TABLE, ROW_ID); + + fakeRow.data = { + a: 'a', + b: 'b' + }; + + row.parent.getRows = function(r, callback) { + callback(null, [fakeRow], response); + }; + + row.get(function(err, row_, apiResponse) { + assert.ifError(err); + assert.strictEqual(row_, row); + assert.deepEqual(row.data, fakeRow.data); + assert.strictEqual(response, apiResponse); + done(); + }); + }); + + it('should return only data for the keys provided', function(done) { + var response = {}; + var fakeRow = new Row(TABLE, ROW_ID); + + fakeRow.data = { + a: 'a', + b: 'b' + }; + + var keys = ['a', 'b']; + + row.data = { + c: 'c' + }; + + row.parent.getRows = function(r, callback) { + callback(null, [fakeRow], response); + }; + + row.get(keys, function(err, data, apiResponse) { + assert.ifError(err); + assert.deepEqual(Object.keys(data), keys); + assert.strictEqual(response, apiResponse); + done(); + }); + }); + }); + + describe('getMetadata', function() { + it('should return an error to the callback', function(done) { + var error = new Error('err'); + var response = {}; + + row.get = function(callback) { + callback(error, null, response); + }; + + row.getMetadata(function(err, metadata, apiResponse) { + assert.strictEqual(error, err); + assert.strictEqual(metadata, null); + assert.strictEqual(response, apiResponse); + done(); + }); + }); + + it('should return metadata to the callback', function(done) { + var response = {}; + var metadata = { + a: 'a', + b: 'b' + }; + + row.get = function(callback) { + row.metadata = metadata; + callback(null, row, response); + }; + + row.getMetadata(function(err, metadata, apiResponse) { + assert.ifError(err); + assert.strictEqual(metadata, metadata); + assert.strictEqual(response, apiResponse); + done(); + }); + }); + }); + + describe('increment', function() { + var COLUMN_NAME = 'a:b'; + var formatFamiliesSpy; + + beforeEach(function() { + formatFamiliesSpy = sinon.stub(Row, 'formatFamilies_').returns({ + a: { + b: [{ + value: 10 + }] + } + }); + }); + + it('should provide the proper request options', function(done) { + row.createRules = function(reqOpts) { + assert.strictEqual(reqOpts.column, COLUMN_NAME); + assert.strictEqual(reqOpts.increment, 1); + done(); + }; + + row.increment(COLUMN_NAME, assert.ifError); + }); + + it('should optionally accept an increment amount', function(done) { + var increment = 10; + + row.createRules = function(reqOpts) { + assert.strictEqual(reqOpts.column, COLUMN_NAME); + assert.strictEqual(reqOpts.increment, increment); + done(); + }; + + row.increment(COLUMN_NAME, increment, assert.ifError); + }); + + it('should return an error to the callback', function(done) { + var error = new Error('err'); + var response = {}; + + row.createRules = function(r, callback) { + callback(error, response); + }; + + row.increment(COLUMN_NAME, function(err, value, apiResponse) { + assert.strictEqual(err, error); + assert.strictEqual(value, null); + assert.strictEqual(apiResponse, response); + done(); + }); + }); + + it('should pass back the updated value to the callback', function(done) { + var fakeValue = 10; + var response = { + key: 'fakeKey', + families: [{ + name: 'a', + columns: [{ + qualifier: 'b', + cells: [{ + timestampMicros: Date.now(), + value: fakeValue, + labels: [] + }] + }] + }] + }; + + row.createRules = function(r, callback) { + callback(null, response); + }; + + row.increment(COLUMN_NAME, function(err, value, apiResponse) { + assert.ifError(err); + assert.strictEqual(value, fakeValue); + assert.strictEqual(apiResponse, response); + assert.strictEqual(formatFamiliesSpy.callCount, 1); + assert(formatFamiliesSpy.calledWithExactly(response.families)); + done(); + }); + }); + }); + + describe('save', function() { + it('should insert a key value pair', function(done) { + var key = 'a:b'; + var value = 'c'; + + var expectedData = { + d: { + e: 'c' + } + }; + + var parseSpy = Mutation.parseColumnName = sinon.spy(function() { + return { + family: 'd', + qualifier: 'e' + }; + }); + + row.parent.mutate = function(entry, callback) { + assert.strictEqual(entry.key, ROW_ID); + assert.deepEqual(entry.data, expectedData); + assert.strictEqual(entry.method, FakeMutation.methods.INSERT); + assert(parseSpy.calledWithExactly(key)); + callback(); + }; + + row.save(key, value, done); + }); + + it('should insert an object', function(done) { + var data = { + a: { + b: 'c' + } + }; + + row.parent.mutate = function(entry) { + assert.strictEqual(entry.data, data); + done(); + }; + + row.save(data, assert.ifError); + }); + }); + + describe('RowError', function() { + it('should supply the correct message', function() { + var error = new Row.RowError('test'); + assert.strictEqual(error.message, 'Unknown row: test.'); + }); + }); + +}); diff --git a/test/bigtable/table.js b/test/bigtable/table.js new file mode 100644 index 00000000000..b85b3d6f8d5 --- /dev/null +++ b/test/bigtable/table.js @@ -0,0 +1,941 @@ +/** + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var assert = require('assert'); +var mockery = require('mockery-next'); +var nodeutil = require('util'); +var pumpify = require('pumpify'); +var through = require('through2'); +var Stream = require('stream').PassThrough; +var sinon = require('sinon').sandbox.create(); + +var GrpcServiceObject = require('../../lib/common/grpc-service-object.js'); +var Family = require('../../lib/bigtable/family.js'); +var Mutation = require('../../lib/bigtable/mutation.js'); +var Row = require('../../lib/bigtable/row.js'); + +function FakeGrpcServiceObject() { + this.calledWith_ = arguments; + GrpcServiceObject.apply(this, arguments); +} + +nodeutil.inherits(FakeGrpcServiceObject, GrpcServiceObject); + +function FakeFamily() { + this.calledWith_ = arguments; + Family.apply(this, arguments); +} + +FakeFamily.formatRule_ = sinon.spy(function(rule) { + return rule; +}); + +nodeutil.inherits(FakeFamily, Family); + +function FakeRow() { + this.calledWith_ = arguments; + Row.apply(this, arguments); +} + +FakeRow.formatChunks_ = sinon.spy(function(chunks) { + return chunks; +}); + +nodeutil.inherits(FakeRow, Row); + +var FakeMutation = { + methods: Mutation.methods, + convertToBytes: sinon.spy(function(value) { + return value; + }), + convertFromBytes: sinon.spy(function(value) { + return value; + }), + parse: sinon.spy(function(value) { + return value; + }) +}; + +var FakeFilter = { + parse: sinon.spy(function(value) { + return value; + }) +}; + +describe('Bigtable/Table', function() { + var TABLE_ID = 'my-table'; + var BIGTABLE = { + clusterName: 'a/b/c/d' + }; + var TABLE_NAME = BIGTABLE.clusterName + '/tables/' + TABLE_ID; + + var Table; + var table; + + before(function() { + mockery.registerMock( + '../../lib/common/grpc-service-object', FakeGrpcServiceObject); + mockery.registerMock('../../lib/bigtable/family', FakeFamily); + mockery.registerMock('../../lib/bigtable/mutation', FakeMutation); + mockery.registerMock('../../lib/bigtable/filter', FakeFilter); + mockery.registerMock('pumpify', pumpify); + mockery.registerMock('../../lib/bigtable/row', FakeRow); + + mockery.enable({ + useCleanCache: true, + warnOnUnregistered: false + }); + + Table = require('../../lib/bigtable/table.js'); + }); + + after(function() { + mockery.deregisterAll(); + mockery.disable(); + }); + + beforeEach(function() { + table = new Table(BIGTABLE, TABLE_ID); + }); + + afterEach(function() { + sinon.restore(); + + Object.keys(FakeMutation).forEach(function(spy) { + if (FakeMutation[spy].reset) { + FakeMutation[spy].reset(); + } + }); + + FakeFilter.parse.reset(); + }); + + describe('instantiation', function() { + it('should inherit from GrpcServiceObject', function() { + var FAKE_TABLE_NAME = 'fake-table-name'; + + sinon.stub(Table, 'formatName_', function() { + return FAKE_TABLE_NAME; + }); + + var table = new Table(BIGTABLE, TABLE_ID); + var config = table.calledWith_[0]; + + assert(table instanceof FakeGrpcServiceObject); + assert.strictEqual(config.parent, BIGTABLE); + assert.strictEqual(config.id, FAKE_TABLE_NAME); + + assert.deepEqual(config.methods, { + create: true, + delete: { + protoOpts: { + service: 'BigtableTableService', + method: 'deleteTable' + }, + reqOpts: { + name: FAKE_TABLE_NAME + } + }, + exists: true, + get: true, + getMetadata: { + protoOpts: { + service: 'BigtableTableService', + method: 'getTable' + }, + reqOpts: { + name: FAKE_TABLE_NAME + } + } + }); + + assert(Table.formatName_.calledWith(BIGTABLE.clusterName, TABLE_ID)); + }); + + it('should use Bigtable#createTable to create the table', function(done) { + var fakeOptions = {}; + + BIGTABLE.createTable = function(name, options, callback) { + assert.strictEqual(name, TABLE_ID); + assert.strictEqual(options, fakeOptions); + callback(); + }; + + table.createMethod(null, fakeOptions, done); + }); + }); + + describe('formatName_', function() { + it('should format the table name to include the cluster name', function() { + var tableName = Table.formatName_(BIGTABLE.clusterName, TABLE_ID); + assert.strictEqual(tableName, TABLE_NAME); + }); + + it('should not re-format the table name', function() { + var tableName = Table.formatName_(BIGTABLE.clusterName, TABLE_NAME); + assert.strictEqual(tableName, TABLE_NAME); + }); + }); + + describe('formatRowRange_', function() { + it('should create a row range object', function() { + var fakeRange = { + start: 'a', + end: 'b' + }; + var convertedFakeRange = { + start: 'c', + end: 'd' + }; + + var spy = FakeMutation.convertToBytes = sinon.spy(function(value) { + if (value === fakeRange.start) { + return convertedFakeRange.start; + } + return convertedFakeRange.end; + }); + + var range = Table.formatRowRange_(fakeRange); + + assert.deepEqual(range, { + startKey: convertedFakeRange.start, + endKey: convertedFakeRange.end + }); + + assert.strictEqual(spy.callCount, 2); + assert.strictEqual(spy.getCall(0).args[0], 'a'); + assert.strictEqual(spy.getCall(1).args[0], 'b'); + }); + }); + + describe('createFamily', function() { + var COLUMN_ID = 'my-column'; + + it('should provide the proper request options', function(done) { + table.request = function(grpcOpts, reqOpts) { + assert.deepEqual(grpcOpts, { + service: 'BigtableTableService', + method: 'createColumnFamily' + }); + + assert.strictEqual(reqOpts.name, TABLE_NAME); + assert.strictEqual(reqOpts.columnFamilyId, COLUMN_ID); + done(); + }; + + table.createFamily(COLUMN_ID, assert.ifError); + }); + + it('should respect the gc expression option', function(done) { + var expression = 'a && b'; + + table.request = function(g, reqOpts) { + assert.strictEqual(reqOpts.columnFamily.gcExpression, expression); + done(); + }; + + table.createFamily(COLUMN_ID, expression, assert.ifError); + }); + + it('should respect the gc rule option', function(done) { + var rule = { + a: 'a', + b: 'b' + }; + var convertedRule = { + c: 'c', + d: 'd' + }; + + var spy = FakeFamily.formatRule_ = sinon.spy(function() { + return convertedRule; + }); + + table.request = function(g, reqOpts) { + assert.strictEqual(reqOpts.columnFamily.gcRule, convertedRule); + assert.strictEqual(spy.callCount, 1); + assert.strictEqual(spy.getCall(0).args[0], rule); + done(); + }; + + table.createFamily(COLUMN_ID, rule, assert.ifError); + }); + + it('should return an error to the callback', function(done) { + var error = new Error('err'); + var response = {}; + + table.request = function(g, r, callback) { + callback(error, response); + }; + + table.createFamily(COLUMN_ID, function(err, family, apiResponse) { + assert.strictEqual(error, err); + assert.strictEqual(family, null); + assert.strictEqual(response, apiResponse); + done(); + }); + }); + + it('should return a Family object', function(done) { + var response = { + name: 'response-family-name' + }; + var fakeFamily = {}; + + table.request = function(g, r, callback) { + callback(null, response); + }; + + table.family = function(name) { + assert.strictEqual(name, response.name); + return fakeFamily; + }; + + table.createFamily(COLUMN_ID, function(err, family, apiResponse) { + assert.ifError(err); + assert.strictEqual(family, fakeFamily); + assert.strictEqual(family.metadata, response); + assert.strictEqual(apiResponse, response); + done(); + }); + }); + }); + + describe('deleteRows', function() { + it('should provide the proper request options', function(done) { + table.request = function(grpcOpts, reqOpts, callback) { + assert.deepEqual(grpcOpts, { + service: 'BigtableTableService', + method: 'bulkDeleteRows' + }); + + assert.strictEqual(reqOpts.tableName, TABLE_NAME); + callback(); + }; + + table.deleteRows(done); + }); + + it('should respect the row key prefix option', function(done) { + var options = { + prefix: 'a' + }; + var fakePrefix = 'b'; + + var spy = FakeMutation.convertToBytes = sinon.spy(function() { + return fakePrefix; + }); + + table.request = function(g, reqOpts, callback) { + assert.strictEqual(reqOpts.rowKeyPrefix, fakePrefix); + + assert.strictEqual(spy.callCount, 1); + assert.strictEqual(spy.getCall(0).args[0], options.prefix); + callback(); + }; + + table.deleteRows(options, done); + }); + + it('should delete all data when no options are provided', function(done) { + table.request = function(g, reqOpts, callback) { + assert.strictEqual(reqOpts.deleteAllDataFromTable, true); + callback(); + }; + + table.deleteRows(done); + }); + }); + + describe('family', function() { + var FAMILY_ID = 'test-family'; + + it('should create a family with the proper arguments', function() { + var family = table.family(FAMILY_ID); + + assert(family instanceof FakeFamily); + assert.strictEqual(family.calledWith_[0], table); + assert.strictEqual(family.calledWith_[1], FAMILY_ID); + }); + }); + + describe('getFamilies', function() { + it('should return an error to the callback', function(done) { + var error = new Error('err'); + var response = {}; + + table.getMetadata = function(callback) { + callback(error, response); + }; + + table.getFamilies(function(err, families, apiResponse) { + assert.strictEqual(err, error); + assert.strictEqual(families, null); + assert.strictEqual(response, apiResponse); + done(); + }); + }); + + it('should return an array of Family objects', function(done) { + var metadata = { + a: 'b' + }; + + var response = { + columnFamilies: { + test: metadata + } + }; + + var fakeFamily = {}; + + table.getMetadata = function(callback) { + callback(null, response); + }; + + table.family = function(id) { + assert.strictEqual(id, 'test'); + return fakeFamily; + }; + + table.getFamilies(function(err, families, apiResponse) { + assert.ifError(err); + + var family = families[0]; + assert.strictEqual(family, fakeFamily); + assert.strictEqual(family.metadata, metadata); + assert.strictEqual(response, apiResponse); + done(); + }); + }); + }); + + describe('getRows', function() { + describe('options', function() { + var pumpSpy; + var formatSpy; + + beforeEach(function() { + pumpSpy = sinon.stub(pumpify, 'obj', function() { + return through.obj(); + }); + + formatSpy = sinon.stub(Table, 'formatRowRange_', function(value) { + return value; + }); + }); + + it('should provide the proper request options', function(done) { + table.requestStream = function(grpcOpts, reqOpts) { + assert.deepEqual(grpcOpts, { + service: 'BigtableService', + method: 'readRows' + }); + + assert.strictEqual(reqOpts.tableName, TABLE_NAME); + assert.strictEqual(reqOpts.objectMode, true); + done(); + }; + + table.getRows(); + }); + + it('should retrieve an individual row', function(done) { + var options = { + key: 'gwashington' + }; + var fakeKey = 'a'; + + var convertSpy = FakeMutation.convertToBytes = sinon.spy(function() { + return fakeKey; + }); + + table.requestStream = function(g, reqOpts) { + assert.strictEqual(reqOpts.rowKey, fakeKey); + assert.strictEqual(convertSpy.callCount, 1); + assert.strictEqual(convertSpy.getCall(0).args[0], options.key); + done(); + }; + + table.getRows(options); + }); + + it('should retrieve a range of rows', function(done) { + var options = { + start: 'gwashington', + end: 'alincoln' + }; + + var fakeRange = { + start: 'a', + end: 'b' + }; + + var formatSpy = Table.formatRowRange_ = sinon.spy(function() { + return fakeRange; + }); + + table.requestStream = function(g, reqOpts) { + assert.strictEqual(reqOpts.rowRange, fakeRange); + assert.strictEqual(formatSpy.callCount, 1); + assert.strictEqual(formatSpy.getCall(0).args[0], options); + done(); + }; + + table.getRows(options); + }); + + it('should retrieve multiple rows', function(done) { + var options = { + keys: [ + 'gwashington', + 'alincoln' + ] + }; + var convertedKeys = [ + 'a', + 'b' + ]; + + var convertSpy = FakeMutation.convertToBytes = sinon.spy(function(key) { + var keyIndex = options.keys.indexOf(key); + return convertedKeys[keyIndex]; + }); + + table.requestStream = function(g, reqOpts) { + assert.deepEqual(reqOpts.rowSet.rowKeys, convertedKeys); + assert.strictEqual(convertSpy.callCount, 2); + assert.strictEqual(convertSpy.getCall(0).args[0], options.keys[0]); + assert.strictEqual(convertSpy.getCall(1).args[0], options.keys[1]); + done(); + }; + + table.getRows(options); + }); + + it('should retrieve multiple ranges', function(done) { + var options = { + ranges: [{ + start: 'a', + end: 'b' + }, { + start: 'c', + end: 'd' + }] + }; + + var fakeRanges = [{ + start: 'e', + end: 'f' + }, { + start: 'g', + end: 'h' + }]; + + var formatSpy = Table.formatRowRange_ = sinon.spy(function(range) { + var rangeIndex = options.ranges.indexOf(range); + return fakeRanges[rangeIndex]; + }); + + table.requestStream = function(g, reqOpts) { + assert.deepEqual(reqOpts.rowSet.rowRanges, fakeRanges); + assert.strictEqual(formatSpy.callCount, 2); + assert.strictEqual(formatSpy.getCall(0).args[0], options.ranges[0]); + assert.strictEqual(formatSpy.getCall(1).args[0], options.ranges[1]); + done(); + }; + + table.getRows(options); + }); + + it('should parse a filter object', function(done) { + var options = { + filter: [{}] + }; + + var fakeFilter = {}; + + var parseSpy = FakeFilter.parse = sinon.spy(function() { + return fakeFilter; + }); + + table.requestStream = function(g, reqOpts) { + assert.strictEqual(reqOpts.filter, fakeFilter); + assert.strictEqual(parseSpy.callCount, 1); + assert.strictEqual(parseSpy.getCall(0).args[0], options.filter); + done(); + }; + + table.getRows(options); + }); + + it('should allow row interleaving', function(done) { + var options = { + interleave: true + }; + + table.requestStream = function(g, reqOpts) { + assert.strictEqual(reqOpts.allowRowInterleaving, options.interleave); + done(); + }; + + table.getRows(options); + }); + + it('should allow setting a row limit', function(done) { + var options = { + limit: 10 + }; + + table.requestStream = function(g, reqOpts) { + assert.strictEqual(reqOpts.numRowsLimit, options.limit); + done(); + }; + + table.getRows(options); + }); + }); + + describe('success', function() { + var fakeRows = [{ + rowKey: 'a', + chunks: [] + }, { + rowKey: 'b', + chunks: [] + }]; + var convertedKeys = [ + 'c', + 'd' + ]; + var formattedChunks = [ + [{ a: 'a' }], + [{ b: 'b' }] + ]; + + beforeEach(function() { + sinon.stub(table, 'row', function() { + return {}; + }); + + FakeMutation.convertFromBytes = sinon.spy(function(value) { + if (value === fakeRows[0].rowKey) { + return convertedKeys[0]; + } + return convertedKeys[1]; + }); + + FakeRow.formatChunks_ = sinon.spy(function(value) { + if (value === fakeRows[0].chunks) { + return formattedChunks[0]; + } + return formattedChunks[1]; + }); + + table.requestStream = function() { + var stream = new Stream({ + objectMode: true + }); + + setImmediate(function() { + fakeRows.forEach(function(data) { + stream.push(data); + }); + + stream.push(null); + }); + + return stream; + }; + }); + + it('should stream Row objects', function(done) { + var rows = []; + + table.getRows() + .on('error', done) + .on('data', function(row) { + rows.push(row); + }) + .on('end', function() { + var rowSpy = table.row; + var formatSpy = FakeRow.formatChunks_; + + assert.strictEqual(rows.length, fakeRows.length); + assert.strictEqual(rowSpy.callCount, fakeRows.length); + + assert.strictEqual(rowSpy.getCall(0).args[0], convertedKeys[0]); + assert.strictEqual(rows[0].data, formattedChunks[0]); + assert.strictEqual( + formatSpy.getCall(0).args[0], fakeRows[0].chunks); + + assert.strictEqual(rowSpy.getCall(1).args[0], convertedKeys[1]); + assert.strictEqual(rows[1].data, formattedChunks[1]); + assert.strictEqual( + formatSpy.getCall(1).args[0], fakeRows[1].chunks); + + done(); + }); + }); + + it('should return an array of Row objects via callback', function(done) { + table.getRows(function(err, rows) { + assert.ifError(err); + + var rowSpy = table.row; + var formatSpy = FakeRow.formatChunks_; + + assert.strictEqual(rows.length, fakeRows.length); + assert.strictEqual(rowSpy.callCount, fakeRows.length); + + assert.strictEqual(rowSpy.getCall(0).args[0], convertedKeys[0]); + assert.strictEqual(rows[0].data, formattedChunks[0]); + assert.strictEqual( + formatSpy.getCall(0).args[0], fakeRows[0].chunks); + + assert.strictEqual(rowSpy.getCall(1).args[0], convertedKeys[1]); + assert.strictEqual(rows[1].data, formattedChunks[1]); + assert.strictEqual( + formatSpy.getCall(1).args[0], fakeRows[1].chunks); + + done(); + }); + }); + }); + + describe('error', function() { + var error = new Error('err'); + + beforeEach(function() { + table.requestStream = function() { + var stream = new Stream({ + objectMode: true + }); + + setImmediate(function() { + stream.emit('error', error); + }); + + return stream; + }; + }); + + it('should emit an error event', function(done) { + table.getRows() + .on('error', function(err) { + assert.strictEqual(error, err); + done(); + }) + .on('data', done); + }); + + it('should return an error to the callback', function(done) { + table.getRows(function(err, rows) { + assert.strictEqual(error, err); + assert(!rows); + done(); + }); + }); + }); + }); + + describe('insert', function() { + it('should create an "insert" mutation', function(done) { + var fakeEntries = [{ + key: 'a', + data: {} + }, { + key: 'b', + data: {} + }]; + + table.mutate = function(entries, callback) { + assert.deepEqual(entries[0], { + key: fakeEntries[0].key, + data: fakeEntries[0].data, + method: FakeMutation.methods.INSERT + }); + + assert.deepEqual(entries[1], { + key: fakeEntries[1].key, + data: fakeEntries[1].data, + method: FakeMutation.methods.INSERT + }); + + callback(); + }; + + table.insert(fakeEntries, done); + }); + }); + + describe('mutate', function() { + it('should provide the proper request options', function(done) { + var entries = [{}, {}]; + var fakeEntries = [{}, {}]; + + var parseSpy = FakeMutation.parse = sinon.spy(function(value) { + var entryIndex = entries.indexOf(value); + return fakeEntries[entryIndex]; + }); + + table.request = function(grpcOpts, reqOpts, callback) { + assert.deepEqual(grpcOpts, { + service: 'BigtableService', + method: 'mutateRows' + }); + + assert.strictEqual(reqOpts.tableName, TABLE_NAME); + assert.deepEqual(reqOpts.entries, fakeEntries); + + assert.strictEqual(parseSpy.callCount, 2); + assert.strictEqual(parseSpy.getCall(0).args[0], entries[0]); + assert.strictEqual(parseSpy.getCall(1).args[0], entries[1]); + callback(); + }; + + table.mutate(entries, done); + }); + }); + + describe('row', function() { + var ROW_ID = 'test-row'; + + it('should return a Row object', function() { + var row = table.row(ROW_ID); + + assert(row instanceof FakeRow); + assert.strictEqual(row.calledWith_[0], table); + assert.strictEqual(row.calledWith_[1], ROW_ID); + }); + }); + + describe('sampleRowKeys', function() { + it('should provide the proper request options', function(done) { + table.requestStream = function(grpcOpts, reqOpts) { + assert.deepEqual(grpcOpts, { + service: 'BigtableService', + method: 'sampleRowKeys' + }); + + assert.strictEqual(reqOpts.tableName, TABLE_NAME); + assert.strictEqual(reqOpts.objectMode, true); + setImmediate(done); + + return new Stream({ + objectMode: true + }); + }; + + table.sampleRowKeys(); + }); + + describe('success', function() { + var fakeKeys = [{ + rowKey: 'a', + offsetBytes: 10 + }, { + rowKey: 'b', + offsetByte: 20 + }]; + + beforeEach(function() { + table.requestStream = function() { + var stream = new Stream({ + objectMode: true + }); + + setImmediate(function() { + fakeKeys.forEach(function(key) { + stream.push(key); + }); + + stream.push(null); + }); + + return stream; + }; + }); + + it('should stream key objects', function(done) { + var keys = []; + + table.sampleRowKeys() + .on('error', done) + .on('data', function(key) { + keys.push(key); + }) + .on('end', function() { + assert.strictEqual(keys[0].key, fakeKeys[0].rowKey); + assert.strictEqual(keys[0].offset, fakeKeys[0].offsetBytes); + assert.strictEqual(keys[1].key, fakeKeys[1].rowKey); + assert.strictEqual(keys[1].offset, fakeKeys[1].offsetBytes); + done(); + }); + }); + + it('should return an array of keys via callback', function(done) { + table.sampleRowKeys(function(err, keys) { + assert.ifError(err); + assert.strictEqual(keys[0].key, fakeKeys[0].rowKey); + assert.strictEqual(keys[0].offset, fakeKeys[0].offsetBytes); + assert.strictEqual(keys[1].key, fakeKeys[1].rowKey); + assert.strictEqual(keys[1].offset, fakeKeys[1].offsetBytes); + done(); + }); + }); + }); + + describe('error', function() { + var error = new Error('err'); + + beforeEach(function() { + table.requestStream = function() { + var stream = new Stream({ + objectMode: true + }); + + setImmediate(function() { + stream.emit('error', error); + }); + + return stream; + }; + }); + + it('should emit an error event', function(done) { + table.sampleRowKeys() + .on('error', function(err) { + assert.strictEqual(err, error); + done(); + }) + .on('data', done); + }); + + it('should return an error to the callback', function(done) { + table.sampleRowKeys(function(err, keys) { + assert.strictEqual(error, err); + assert(!keys); + done(); + }); + }); + }); + }); + +}); diff --git a/test/common/grpc-service-object.js b/test/common/grpc-service-object.js index c40dbb73f68..4907fa84b4e 100644 --- a/test/common/grpc-service-object.js +++ b/test/common/grpc-service-object.js @@ -156,4 +156,18 @@ describe('GrpcServiceObject', function() { grpcServiceObject.request(PROTO_OPTS, REQ_OPTS, done); }); }); + + describe('requestStream', function() { + it('should call the parent instance requestStream method', function(done) { + grpcServiceObject.parent = { + requestStream: function(protoOpts, reqOpts) { + assert.strictEqual(protoOpts, PROTO_OPTS); + assert.strictEqual(reqOpts, REQ_OPTS); + done(); + } + }; + + grpcServiceObject.requestStream(PROTO_OPTS, REQ_OPTS); + }); + }); }); diff --git a/test/common/grpc-service.js b/test/common/grpc-service.js index 5f2ae1c85b5..6f30a3563a0 100644 --- a/test/common/grpc-service.js +++ b/test/common/grpc-service.js @@ -24,6 +24,7 @@ var is = require('is'); var mockery = require('mockery-next'); var path = require('path'); var retryRequest = require('retry-request'); +var through = require('through2'); var util = require('../../lib/common/util.js'); @@ -763,6 +764,30 @@ describe('GrpcService', function() { // the callback passed to retry-request. We will check if the grpc Error retryRequestOptions.request({}, retryRequestCallback); }); + + it('should exec callback with unknown error', function(done) { + var unknownError = { a: 'a' }; + + grpcService.protos.Service = { + service: function() { + return { + method: function(reqOpts, grpcOpts, callback) { + callback(unknownError, null); + } + }; + } + }; + + grpcService.request(PROTO_OPTS, REQ_OPTS, function(err, resp) { + assert.strictEqual(err, unknownError); + assert.strictEqual(resp, null); + done(); + }); + + // When the gRPC error is passed to "onResponse", it will just invoke + // the callback passed to retry-request. We will check if the grpc Error + retryRequestOptions.request({}, retryRequestCallback); + }); }); it('should make the correct request on the proto service', function(done) { @@ -841,7 +866,6 @@ describe('GrpcService', function() { }; grpcService.request(PROTO_OPTS, REQ_OPTS, function(err) { - assert.strictEqual(err, grpcError); assert.strictEqual(err.code, httpError.code); }); } @@ -874,6 +898,272 @@ describe('GrpcService', function() { }); }); + describe('requestStream', function() { + var PROTO_OPTS = { service: 'service', method: 'method', timeout: 3000 }; + var REQ_OPTS = {}; + var GRPC_CREDENTIALS = {}; + var fakeStream; + + function ProtoService() {} + + beforeEach(function() { + ProtoService.prototype.method = function() {}; + + grpcService.grpcCredentials = GRPC_CREDENTIALS; + grpcService.baseUrl = 'http://base-url'; + grpcService.proto = {}; + grpcService.proto.service = ProtoService; + + grpcService.getService_ = function() { + return new ProtoService(); + }; + + fakeStream = through.obj(); + retryRequestOverride = function() { + return fakeStream; + }; + }); + + afterEach(function() { + retryRequestOverride = null; + }); + + it('should not run in the gcloud sandbox environment', function() { + delete grpcService.grpcCredentials; + + grpcService.getGrpcCredentials_ = function() { + throw new Error('Should not be called.'); + }; + + global.GCLOUD_SANDBOX_ENV = true; + grpcService.requestStream(); + + delete global.GCLOUD_SANDBOX_ENV; + }); + + it('should get the proto service', function(done) { + grpcService.getService_ = function(protoOpts) { + assert.strictEqual(protoOpts, PROTO_OPTS); + setImmediate(done); + return new ProtoService(); + }; + + grpcService.requestStream(PROTO_OPTS, REQ_OPTS, assert.ifError); + }); + + it('should set the deadline', function(done) { + var createDeadline = GrpcService.createDeadline_; + var fakeDeadline = createDeadline(PROTO_OPTS.timeout); + + GrpcService.createDeadline_ = function(timeout) { + assert.strictEqual(timeout, PROTO_OPTS.timeout); + return fakeDeadline; + }; + + ProtoService.prototype.method = function(reqOpts, grpcOpts) { + assert.strictEqual(grpcOpts.deadline, fakeDeadline); + + GrpcService.createDeadline_ = createDeadline; + setImmediate(done); + + return through.obj(); + }; + + retryRequestOverride = function(_, retryOpts) { + return retryOpts.request(); + }; + + grpcService.requestStream(PROTO_OPTS, REQ_OPTS); + }); + + describe('getting gRPC credentials', function() { + beforeEach(function() { + delete grpcService.grpcCredentials; + }); + + describe('error', function() { + var error = new Error('err'); + + beforeEach(function() { + grpcService.getGrpcCredentials_ = function(callback) { + callback(error); + }; + }); + + it('should execute callback with error', function(done) { + grpcService.requestStream(PROTO_OPTS, REQ_OPTS) + .on('error', function(err) { + assert.strictEqual(err, error); + done(); + }); + }); + }); + + describe('success', function() { + var authClient = {}; + + beforeEach(function() { + grpcService.getGrpcCredentials_ = function(callback) { + callback(null, authClient); + }; + }); + + it('should make the gRPC request again', function(done) { + grpcService.getService_ = function() { + assert.strictEqual(grpcService.grpcCredentials, authClient); + setImmediate(done); + return new ProtoService(); + }; + + grpcService.requestStream(PROTO_OPTS, REQ_OPTS) + .on('error', done); + }); + }); + }); + + describe('retry strategy', function() { + var retryRequestReqOpts; + var retryRequestOptions; + var retryStream; + + beforeEach(function() { + retryRequestReqOpts = retryRequestOptions = null; + retryStream = through.obj(); + + retryRequestOverride = function(reqOpts, options) { + retryRequestReqOpts = reqOpts; + retryRequestOptions = options; + return retryStream; + }; + }); + + afterEach(function() { + retryRequestOverride = null; + }); + + it('should use retry-request', function() { + var reqOpts = extend({ + objectMode: true + }, REQ_OPTS); + + grpcService.requestStream(PROTO_OPTS, reqOpts); + + assert.strictEqual(retryRequestReqOpts, null); + assert.strictEqual( + retryRequestOptions.retries, + grpcService.maxRetries + ); + assert.strictEqual(retryRequestOptions.objectMode, true); + assert.strictEqual( + retryRequestOptions.shouldRetryFn, + GrpcService.shouldRetryRequest_ + ); + }); + + it('should emit the status as a response event', function(done) { + var grpcError500 = { code: 2 }; + var fakeStream = through.obj(); + + ProtoService.prototype.method = function() { + return fakeStream; + }; + + retryRequestOverride = function(reqOpts, options) { + return options.request(); + }; + + fakeStream + .on('response', function(resp) { + assert.deepEqual(resp, GrpcService.GRPC_ERROR_CODE_TO_HTTP[2]); + done(); + }); + + grpcService.requestStream(PROTO_OPTS, REQ_OPTS); + fakeStream.emit('status', grpcError500); + }); + + it('should emit the response error', function(done) { + var grpcError500 = { code: 2 }; + var requestStream = grpcService.requestStream(PROTO_OPTS, REQ_OPTS); + + requestStream.destroy = function(err) { + assert.deepEqual(err, GrpcService.GRPC_ERROR_CODE_TO_HTTP[2]); + done(); + }; + + retryStream.emit('error', grpcError500); + }); + }); + }); + + describe('createDeadline_', function() { + var nowTimestamp = Date.now(); + var now; + + before(function() { + now = Date.now; + + Date.now = function() { + return nowTimestamp; + }; + }); + + after(function() { + Date.now = now; + }); + + it('should create a deadline', function() { + var timeout = 3000; + var deadline = GrpcService.createDeadline_(timeout); + + assert.strictEqual(deadline.getTime(), nowTimestamp + timeout); + }); + }); + + describe('getError_', function() { + it('should retrieve the HTTP error from the gRPC error map', function() { + var errorMap = GrpcService.GRPC_ERROR_CODE_TO_HTTP; + var codes = Object.keys(errorMap); + + codes.forEach(function(code) { + var error = GrpcService.getError_({ code: code }); + + assert.notStrictEqual(error, errorMap[code]); + assert.deepEqual(error, errorMap[code]); + }); + }); + + it('should return null for unknown errors', function() { + var error = GrpcService.getError_({ code: 9999 }); + + assert.strictEqual(error, null); + }); + }); + + describe('shouldRetryRequest_', function() { + it('should retry on 429, 500, 502, and 503', function() { + var shouldRetryFn = GrpcService.shouldRetryRequest_; + + var retryErrors = [ + { code: 429 }, + { code: 500 }, + { code: 502 }, + { code: 503 } + ]; + + var nonRetryErrors = [ + { code: 200 }, + { code: 401 }, + { code: 404 }, + { code: 409 }, + { code: 412 } + ]; + + assert.strictEqual(retryErrors.every(shouldRetryFn), true); + assert.strictEqual(nonRetryErrors.every(shouldRetryFn), false); + }); + }); + describe('getGrpcCredentials_', function() { it('should get credentials from the auth client', function(done) { grpcService.authClient = { @@ -943,4 +1233,130 @@ describe('GrpcService', function() { }); }); }); + + describe('loadProtoFile_', function() { + var fakeServices = { + google: { + FakeService: { + v1: {} + } + } + }; + + it('should load a proto file', function() { + var fakeProtoConfig = { + path: '/root/dir/path', + service: 'FakeService', + apiVersion: 'v1' + }; + + var fakeMainConfig = { + service: 'OtherFakeService', + apiVersion: 'v2' + }; + + grpcLoadOverride = function(pathOpts, type, grpOpts) { + assert.strictEqual(pathOpts.root, ROOT_DIR); + assert.strictEqual(pathOpts.file, 'path'); + assert.strictEqual(type, 'proto'); + + assert.deepEqual(grpOpts, { + binaryAsBase64: true, + convertFieldsToCamelCase: true + }); + + return fakeServices; + }; + + var service = grpcService.loadProtoFile_(fakeProtoConfig, fakeMainConfig); + assert.strictEqual(service, fakeServices.google.FakeService.v1); + }); + + it('should use the main config if protoConfig is not set', function() { + var fakeProtoConfig = '/root/dir/path'; + + var fakeMainConfig = { + service: 'FakeService', + apiVersion: 'v1' + }; + + grpcLoadOverride = function(pathOpts, type, grpOpts) { + assert.strictEqual(pathOpts.root, ROOT_DIR); + assert.strictEqual(pathOpts.file, 'path'); + assert.strictEqual(type, 'proto'); + + assert.deepEqual(grpOpts, { + binaryAsBase64: true, + convertFieldsToCamelCase: true + }); + + return fakeServices; + }; + + var service = grpcService.loadProtoFile_(fakeProtoConfig, fakeMainConfig); + assert.strictEqual(service, fakeServices.google.FakeService.v1); + }); + }); + + describe('getService_', function() { + it('should get a new service instance', function() { + var fakeService = {}; + + grpcService.protos = { + Service: { + Service: function(baseUrl, grpcCredentials) { + assert.strictEqual(baseUrl, grpcService.baseUrl); + assert.strictEqual(grpcCredentials, grpcService.grpcCredentials); + return fakeService; + } + } + }; + + var service = grpcService.getService_({ service: 'Service' }); + assert.strictEqual(service, fakeService); + + var cachedService = grpcService.activeServiceMap_.get('Service'); + assert.strictEqual(cachedService, fakeService); + }); + + it('should return the default service', function() { + var fakeService = {}; + + grpcService.protos = { + Service: { + OtherService: function(baseUrl, grpcCredentials) { + assert.strictEqual(baseUrl, grpcService.baseUrl); + assert.strictEqual(grpcCredentials, grpcService.grpcCredentials); + return fakeService; + } + } + }; + + var service = grpcService.getService_({ service: 'OtherService' }); + assert.strictEqual(service, fakeService); + + var cachedService = grpcService.activeServiceMap_.get('OtherService'); + assert.strictEqual(cachedService, fakeService); + }); + + it('should return the cached version of a service', function() { + var fakeService = {}; + + grpcService.protos = { + Service: { + Service: function() { + throw new Error('should not be called'); + } + } + }; + + grpcService.activeServiceMap_.set('Service', fakeService); + + var service = grpcService.getService_({ service: 'Service' }); + assert.strictEqual(service, fakeService); + + var cachedService = grpcService.activeServiceMap_.get('Service'); + assert.strictEqual(cachedService, fakeService); + }); + }); }); diff --git a/test/index.js b/test/index.js index fb0117a82f7..86caac47bb9 100644 --- a/test/index.js +++ b/test/index.js @@ -33,6 +33,7 @@ function createFakeApi() { } var FakeBigQuery = createFakeApi(); +var FakeBigtable = createFakeApi(); var FakeCompute = createFakeApi(); var FakeDatastore = createFakeApi(); var FakeDNS = createFakeApi(); @@ -47,6 +48,7 @@ describe('gcloud', function() { before(function() { mockery.registerMock('../lib/bigquery', FakeBigQuery); + mockery.registerMock('../lib/bigtable', FakeBigtable); mockery.registerMock('../lib/compute', FakeCompute); mockery.registerMock('../lib/datastore', FakeDatastore); mockery.registerMock('../lib/dns', FakeDNS); @@ -75,6 +77,10 @@ describe('gcloud', function() { assert.strictEqual(gcloud.bigquery, FakeBigQuery); }); + it('should export static bigtable', function() { + assert.strictEqual(gcloud.bigtable, FakeBigtable); + }); + it('should export static compute', function() { assert.strictEqual(gcloud.compute, FakeCompute); }); @@ -147,6 +153,15 @@ describe('gcloud', function() { }); }); + describe('bigtable', function() { + it('should create a new Bigtable', function() { + var bigtable = localGcloud.bigtable(options); + + assert(bigtable instanceof FakeBigtable); + assert.strictEqual(bigtable.calledWith_[0], options); + }); + }); + describe('compute', function() { it('should create a new Compute', function() { var compute = localGcloud.compute(options);