ARROW-1990: [JS] C++ Refactor, Add DataFrame

js/package.json

@@ -12,6 +12,7 @@
  "clean": "gulp clean",
  "debug": "gulp debug",
  "perf": "node ./perf/index.js",
+ "create:perfdata": "python ./test/data/tables/generate.py ./test/data/tables/tracks.arrow",
  "release": "./npm-release.sh",
  "clean:all": "run-p clean clean:testdata",
  "clean:testdata": "gulp clean:testdata",

js/perf/index.js

@@ -16,21 +16,21 @@
 // under the License.
 
 // Use the ES5 UMD target as perf baseline
-// const { Table, readVectors } = require('../targets/es5/umd');
-// const { Table, readVectors } = require('../targets/es5/cjs');
-const { Table, readVectors } = require('../targets/es2015/umd');
-// const { Table, readVectors } = require('../targets/es2015/cjs');
+// const { col, Table, readVectors } = require('../targets/es5/umd');
+// const { col, Table, readVectors } = require('../targets/es5/cjs');
+// const { col, Table, readVectors } = require('../targets/es2015/umd');
+const { col, Table, readVectors } = require('../targets/es2015/cjs');
 
 const config = require('./config');
 const Benchmark = require('benchmark');
 
 const suites = [];
 
 for (let { name, buffers} of config) {
- const parseSuite = new Benchmark.Suite(`Parse ${name}`, { async: true });
- const sliceSuite = new Benchmark.Suite(`Slice ${name} vectors`, { async: true });
- const iterateSuite = new Benchmark.Suite(`Iterate ${name} vectors`, { async: true });
- const getByIndexSuite = new Benchmark.Suite(`Get ${name} values by index`, { async: true });
+ const parseSuite = new Benchmark.Suite(`Parse "${name}"`, { async: true });
+ const sliceSuite = new Benchmark.Suite(`Slice "${name}" vectors`, { async: true });
+ const iterateSuite = new Benchmark.Suite(`Iterate "${name}" vectors`, { async: true });
+ const getByIndexSuite = new Benchmark.Suite(`Get "${name}" values by index`, { async: true });
  parseSuite.add(createFromTableTest(name, buffers));
  parseSuite.add(createReadVectorsTest(name, buffers));
  for (const vector of Table.from(buffers).columns) {
@@ -41,6 +41,25 @@ for (let { name, buffers} of config) {
  suites.push(getByIndexSuite, iterateSuite, sliceSuite, parseSuite);
 }
 
+for (let {name, buffers, countBys, counts} of require('./table_config')) {
+ const table = Table.from(buffers);
+
+ const dfCountBySuite = new Benchmark.Suite(`DataFrame Count By "${name}"`, { async: true });
+ for (countBy of countBys) {
+ dfCountBySuite.add(createDataFrameCountByTest(table, countBy));
+ }
+
+ const dfFilterCountSuite = new Benchmark.Suite(`DataFrame Filter-Scan Count "${name}"`, { async: true });
+ const dfDirectCountSuite = new Benchmark.Suite(`DataFrame Direct Count "${name}"`, { async: true });
+
+ for (test of counts) {
+ dfFilterCountSuite.add(createDataFrameFilterCountTest(table, test.col, test.test, test.value))
+ dfDirectCountSuite.add(createDataFrameDirectCountTest(table, test.col, test.test, test.value))
+ }
+
+ suites.push(dfCountBySuite, dfFilterCountSuite, dfDirectCountSuite)
+}
+
 console.log('Running apache-arrow performance tests...\n');
 
 run();
@@ -109,3 +128,80 @@ function createGetByIndexTest(vector) {
  }
  };
 }
+
+function createDataFrameDirectCountTest(table, column, test, value) {
+ let colidx = table.columns.findIndex((c)=>c.name === column);
+
+ if (test == 'gteq') {
+ op = function () {
+ sum = 0;
+ for (let batch = -1; ++batch < table.lengths.length;) {
+ const length = table.lengths[batch];
+
+ // load batches
+ const columns = table.batches[batch];
+
+ // yield all indices
+ for (let idx = -1; ++idx < length;) {
+ sum += (columns[colidx].get(idx) >= value);
+ }
+ }
+ }
+ } else if (test == 'eq') {
+ op = function() {
+ sum = 0;
+ for (let batch = -1; ++batch < table.lengths.length;) {
+ const length = table.lengths[batch];
+
+ // load batches
+ const columns = table.batches[batch]
+
+ // yield all indices
+ for (let idx = -1; ++idx < length;) {
+ sum += (columns[colidx].get(idx) == value);
+ }
+ }
+ }
+ } else {
+ throw new Error(`Unrecognized test "${test}"`);
+ }
+
+ return {
+ async: true,
+ name: `name: '${column}', length: ${table.length}, type: ${table.columns[colidx].type}, test: ${test}, value: ${value}`,
+ fn: op
+ };
+}
+
+function createDataFrameCountByTest(table, column) {
+ let colidx = table.columns.findIndex((c)=>c.name === column);
+
+ return {
+ async: true,
+ name: `name: '${column}', length: ${table.length}, type: ${table.columns[colidx].type}`,
+ fn() {
+ table.countBy(column);
+ }
+ };
+}
+
+function createDataFrameFilterCountTest(table, column, test, value) {
+ let colidx = table.columns.findIndex((c)=>c.name === column);
+ let df;
+
+ if (test == 'gteq') {
+ df = table.filter(col(column).gteq(value));
+ } else if (test == 'eq') {
+ df = table.filter(col(column).eq(value));
+ } else {
+ throw new Error(`Unrecognized test "${test}"`);
+ }
+
+ return {
+ async: true,
+ name: `name: '${column}', length: ${table.length}, type: ${table.columns[colidx].type}, test: ${test}, value: ${value}`,
+ fn() {
+ df.count();
+ }
+ };
+}

js/perf/table_config.js

@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const fs = require('fs');
+const path = require('path');
+const glob = require('glob');
+
+const config = [];
+const filenames = glob.sync(path.resolve(__dirname, `../test/data/tables/`, `*.arrow`));
+
+countBys = {
+ "tracks": ['origin', 'destination']
+}
+counts = {
+ "tracks": [
+ {col: 'lat', test: 'gteq', value: 0 },
+ {col: 'lng', test: 'gteq', value: 0 },
+ {col: 'origin', test: 'eq', value: 'Seattle'},
+ ]
+}
+
+for (const filename of filenames) {
+ const { name } = path.parse(filename);
+ if (name in counts) {
+ config.push({
+ name,
+ buffers: [fs.readFileSync(filename)],
+ countBys: countBys[name],
+ counts: counts[name],
+ });
+ }
+}
+
+module.exports = config;

js/src/Arrow.externs.ts

@@ -50,6 +50,20 @@ Table.prototype.key;
 Table.prototype.select;
 /** @type {?} */
 Table.prototype.toString;
+/** @type {?} */
+Table.prototype.lengths;
+/** @type {?} */
+Table.prototype.batches;
+/** @type {?} */
+Table.prototype.countBy;
+/** @type {?} */
+Table.prototype.scan;
+/** @type {?} */
+Table.prototype.get;
+
+let CountByResult = function() {};
+/** @type {?} */
+CountByResult.prototype.asJSON;
 
 let Vector = function() {};
 /** @type {?} */
@@ -82,3 +96,11 @@ let DictionaryVector = function() {};
 DictionaryVector.prototype.getKey;
 /** @type {?} */
 DictionaryVector.prototype.getValue;
+
+let Col = function() {};
+/** @type {?} */
+Col.prototype.gteq;
+/** @type {?} */
+Col.prototype.lteq;
+/** @type {?} */
+Col.prototype.eq;

js/src/Arrow.ts

@@ -15,7 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { Table } from './vector/table';
+import { Table, TableRow, CountByResult } from './table';
+import { lit, col, Col, Value } from './predicate';
 import { Vector } from './vector/vector';
 import { Utf8Vector } from './vector/utf8';
 import { DictionaryVector } from './vector/dictionary';
@@ -53,7 +54,9 @@ Table['fromAsync'] = Table.fromAsync;
 BoolVector['pack'] = BoolVector.pack;
 
 export { read, readAsync };
-export { Table, Vector, StructRow };
+export { Table, TableRow, CountByResult };
+export { lit, col, Col, Value };
+export { Vector, StructRow };
 export { Uint64, Int64, Int128 };
 export { NumericVectorConstructor } from './vector/numeric';
 export { List, TypedArray, TypedArrayConstructor } from './vector/types';
@@ -89,9 +92,13 @@ try {
  const Arrow = eval('exports');
  if (typeof Arrow === 'object') {
  // string indexers tell closure compiler not to rename these properties
+ Arrow['lit'] = lit;
+ Arrow['col'] = col;
+ Arrow['Col'] = Col;
  Arrow['read'] = read;
- Arrow['readAsync'] = readAsync;
+ Arrow['Value'] = Value;
  Arrow['Table'] = Table;
+ Arrow['readAsync'] = readAsync;
  Arrow['Vector'] = Vector;
  Arrow['StructRow'] = StructRow;
  Arrow['BoolVector'] = BoolVector;
@@ -115,6 +122,7 @@ try {
  Arrow['Float32Vector'] = Float32Vector;
  Arrow['Float64Vector'] = Float64Vector;
  Arrow['DecimalVector'] = DecimalVector;
+ Arrow['CountByResult'] = CountByResult;
  Arrow['TimestampVector'] = TimestampVector;
  Arrow['DictionaryVector'] = DictionaryVector;
  Arrow['FixedSizeListVector'] = FixedSizeListVector;

js/src/bin/arrow2csv.ts

@@ -97,8 +97,8 @@ files.forEach((source) => {
  printTable(table);
 });
 
-function printTable(table: Arrow.Table<any>) {
- let header = [...table.columns.map((_, i) => table.key(i))].map(stringify);
+function printTable(table: Arrow.Table) {
+ let header = [...table.columns.map((c) => c.name)].map(stringify);
  let maxColumnWidths = header.map(x => x.length);
  // Pass one to convert to strings and count max column widths
  for (let i = -1, n = table.length - 1; ++i < n;) {
@@ -132,4 +132,4 @@ function stringify(x: any) {
  : `${x}`;
 }
 
-})();
+})();