From e537789db9a6b55383a7efabe6353dcb3bb3d7e5 Mon Sep 17 00:00:00 2001 From: Paul Taylor Date: Tue, 23 Jan 2018 16:15:23 -0800 Subject: [PATCH] make it easier to run all integration tests from local data --- js/bin/integration.js | 56 +++++++++++++++++--- js/gulp/argv.js | 6 +-- js/gulp/test-task.js | 6 +-- js/package.json | 8 +-- js/test/integration/validate-tests.ts | 75 +++++++++++++++++---------- 5 files changed, 106 insertions(+), 45 deletions(-) diff --git a/js/bin/integration.js b/js/bin/integration.js index fe32433d3845a..2aeb14d0e3425 100755 --- a/js/bin/integration.js +++ b/js/bin/integration.js @@ -17,6 +17,8 @@ // specific language governing permissions and limitations // under the License. +var fs = require('fs'); +var glob = require('glob'); var path = require('path'); var gulp = require.resolve(path.join(`..`, `node_modules/gulp/bin/gulp.js`)); var child_process = require(`child_process`); @@ -29,12 +31,14 @@ var optionList = [ { type: String, name: 'arrow', alias: 'a', - description: 'The Arrow file to read/write' + multiple: true, defaultValue: [], + description: 'The Arrow file[s] to read/write' }, { type: String, name: 'json', alias: 'j', - description: 'The JSON file to read/write' + multiple: true, defaultValue: [], + description: 'The JSON file[s] to read/write' } ]; @@ -66,20 +70,60 @@ function print_usage() { process.exit(1); } -if (!argv.arrow || !argv.json || !argv.mode) { +let jsonPaths = argv.json; +let arrowPaths = argv.arrow; + +if (!argv.mode) { + return print_usage(); +} + +let mode = argv.mode.toUpperCase(); +if (mode === 'VALIDATE' && !jsonPaths.length) { + jsonPaths = glob.sync(path.resolve(__dirname, `../test/data/json/`, `*.json`)); + if (!arrowPaths.length) { + [jsonPaths, arrowPaths] = jsonPaths.reduce(([jsonPaths, arrowPaths], jsonPath) => { + const { name } = path.parse(jsonPath); + for (const source of ['cpp', 'java']) { + for (const format of ['file', 'stream']) { + const arrowPath = path.resolve(__dirname, `../test/data/${source}/${format}/${name}.arrow`); + if (fs.existsSync(arrowPath)) { + jsonPaths.push(jsonPath); + arrowPaths.push(arrowPath); + console.log('-j', jsonPath, '-a', arrowPath, '\\'); + } + } + } + return [jsonPaths, arrowPaths]; + }, [[], []]); + } +} else if (!jsonPaths.length) { return print_usage(); } -switch (argv.mode.toUpperCase()) { +switch (mode) { case 'VALIDATE': + const args = [`test`, `-i`].concat(argv._unknown || []); + jsonPaths.forEach((p, i) => { + args.push('-j', p, '-a', arrowPaths[i]); + }); child_process.spawnSync( - gulp, - [`test`, `-i`].concat(process.argv.slice(2)), + gulp, args, { cwd: path.resolve(__dirname, '..'), stdio: ['ignore', 'inherit', 'inherit'] } ); + // for (let i = -1, n = jsonPaths.length; ++i < n;) { + // const jsonPath = jsonPaths[i]; + // const arrowPath = arrowPaths[i]; + // child_process.spawnSync( + // gulp, args.concat(['-j', jsonPath, '-a', arrowPath]), + // { + // cwd: path.resolve(__dirname, '..'), + // stdio: ['ignore', 'inherit', 'inherit'] + // } + // ); + // } break; default: print_usage(); diff --git a/js/gulp/argv.js b/js/gulp/argv.js index 6f80912e97e52..4aee2e0fa9396 100644 --- a/js/gulp/argv.js +++ b/js/gulp/argv.js @@ -22,13 +22,11 @@ const argv = require(`command-line-args`)([ { name: `target`, type: String, defaultValue: `` }, { name: `module`, type: String, defaultValue: `` }, { name: `coverage`, type: Boolean, defaultValue: false }, - { name: `json_file`, alias: `j`, type: String, defaultValue: null }, - { name: `arrow_file`, alias: `a`, type: String, defaultValue: null }, { name: `integration`, alias: `i`, type: Boolean, defaultValue: false }, { name: `targets`, alias: `t`, type: String, multiple: true, defaultValue: [] }, { name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] }, - { name: `sources`, alias: `s`, type: String, multiple: true, defaultValue: [`cpp`, `java`] }, - { name: `formats`, alias: `f`, type: String, multiple: true, defaultValue: [`file`, `stream`] }, + { name: `json_files`, alias: `j`, type: String, multiple: true, defaultValue: [] }, + { name: `arrow_files`, alias: `a`, type: String, multiple: true, defaultValue: [] }, ], { partial: true }); const { targets, modules } = argv; diff --git a/js/gulp/test-task.js b/js/gulp/test-task.js index ab280b092635c..885bed0b5ef75 100644 --- a/js/gulp/test-task.js +++ b/js/gulp/test-task.js @@ -48,11 +48,9 @@ const testTask = ((cache, execArgv, testOptions) => memoizeTask(cache, function opts.env = { ...opts.env, TEST_TARGET: target, TEST_MODULE: format, - JSON_PATH: argv.json_file, - ARROW_PATH: argv.arrow_file, TEST_TS_SOURCE: !!argv.coverage, - TEST_SOURCES: JSON.stringify(Array.isArray(argv.sources) ? argv.sources : [argv.sources]), - TEST_FORMATS: JSON.stringify(Array.isArray(argv.formats) ? argv.formats : [argv.formats]), + JSON_PATHS: JSON.stringify(Array.isArray(argv.json_files) ? argv.json_files : [argv.json_files]), + ARROW_PATHS: JSON.stringify(Array.isArray(argv.arrow_files) ? argv.arrow_files : [argv.arrow_files]), }; return !debug ? child_process.spawn(jest, args, opts) : diff --git a/js/package.json b/js/package.json index 38aaebd02ffcd..bc50c77ad5505 100644 --- a/js/package.json +++ b/js/package.json @@ -62,8 +62,8 @@ "devDependencies": { "@std/esm": "0.19.7", "@types/flatbuffers": "1.6.5", - "@types/glob": "5.0.34", - "@types/jest": "22.0.1", + "@types/glob": "5.0.35", + "@types/jest": "22.1.0", "@types/node": "9.3.0", "ast-types": "0.10.1", "benchmark": "2.1.4", @@ -78,9 +78,9 @@ "gulp-rename": "1.2.2", "gulp-sourcemaps": "2.6.3", "gulp-transform-js-ast": "1.0.2", - "gulp-typescript": "3.2.3", + "gulp-typescript": "3.2.4", "ix": "2.3.4", - "jest": "22.1.3", + "jest": "22.1.4", "jest-environment-node-debug": "2.0.0", "json": "9.0.6", "lerna": "2.7.1", diff --git a/js/test/integration/validate-tests.ts b/js/test/integration/validate-tests.ts index 9276ed0279d98..b9d3b1ba481c4 100644 --- a/js/test/integration/validate-tests.ts +++ b/js/test/integration/validate-tests.ts @@ -18,28 +18,41 @@ import * as fs from 'fs'; import * as path from 'path'; -if (!process.env.JSON_PATH || !process.env.ARROW_PATH) { - throw new Error('Integration tests need paths to both json and arrow files'); -} +import Arrow from '../Arrow'; +import { zip } from 'ix/iterable/zip'; +import { toArray } from 'ix/iterable/toarray'; -const jsonPath = path.resolve(process.env.JSON_PATH + ''); -const arrowPath = path.resolve(process.env.ARROW_PATH + ''); +/* tslint:disable */ +const { parse: bignumJSONParse } = require('json-bignum'); -if (!fs.existsSync(jsonPath) || !fs.existsSync(arrowPath)) { - throw new Error('Integration tests need both json and arrow files to exist'); -} +const { Table, read } = Arrow; -/* tslint:disable */ -const { parse } = require('json-bignum'); +if (!process.env.JSON_PATHS || !process.env.ARROW_PATHS) { + throw new Error('Integration tests need paths to both json and arrow files'); +} -const jsonData = parse(fs.readFileSync(jsonPath, 'utf8')); -const arrowBuffers: Uint8Array[] = [fs.readFileSync(arrowPath)]; +function resolvePathArgs(paths: string) { + let pathsArray = JSON.parse(paths) as string | string[]; + return (Array.isArray(pathsArray) ? pathsArray : [pathsArray]) + .map((p) => path.resolve(p)) + .map((p) => { + if (fs.existsSync(p)) { + return p; + } + console.warn(`Could not find file "${p}"`); + return undefined; + }); +} -import Arrow from '../Arrow'; -import { zip } from 'ix/iterable/zip'; -import { toArray } from 'ix/iterable/toarray'; +const getOrReadFileBuffer = ((cache: any) => function getFileBuffer(path: string, ...args: any[]) { + return cache[path] || (cache[path] = fs.readFileSync(path, ...args)); +})({}); -const { Table, read } = Arrow; +const jsonAndArrowPaths = toArray(zip( + resolvePathArgs(process.env.JSON_PATHS!), + resolvePathArgs(process.env.ARROW_PATHS!) +)) +.filter(([p1, p2]) => p1 !== undefined && p2 !== undefined) as [string, string][]; expect.extend({ toEqualVector(v1: any, v2: any) { @@ -66,7 +79,7 @@ expect.extend({ for (let i = -1, n = props.length; ++i < n;) { const prop = props[i]; - if (this.utils.stringify(v1[prop]) !== this.utils.stringify(v2[prop])) { + if (`${v1[prop]}` !== `${v2[prop]}`) { propsFailures.push(`${prop}: ${format(v1[prop], v2[prop], ' !== ')}`); } } @@ -98,35 +111,43 @@ expect.extend({ }); describe(`Integration`, () => { - testReaderIntegration(); - testTableFromBuffersIntegration(); + for (const [jsonFilePath, arrowFilePath] of jsonAndArrowPaths) { + let { name, dir } = path.parse(arrowFilePath); + dir = dir.split(path.sep).slice(-2).join(path.sep); + const json = bignumJSONParse(getOrReadFileBuffer(jsonFilePath, 'utf8')); + const arrowBuffer = getOrReadFileBuffer(arrowFilePath) as Uint8Array; + describe(path.join(dir, name), () => { + testReaderIntegration(json, arrowBuffer); + testTableFromBuffersIntegration(json, arrowBuffer); + }); + } }); -function testReaderIntegration() { - test(`json and arrow buffers report the same values`, () => { - debugger; +function testReaderIntegration(jsonData: any, arrowBuffer: Uint8Array) { + test(`json and arrow record batches report the same values`, () => { expect.hasAssertions(); const jsonRecordBatches = toArray(read(jsonData)); - const binaryRecordBatches = toArray(read(arrowBuffers)); + const binaryRecordBatches = toArray(read(arrowBuffer)); for (const [jsonRecordBatch, binaryRecordBatch] of zip(jsonRecordBatches, binaryRecordBatches)) { expect(jsonRecordBatch.length).toEqual(binaryRecordBatch.length); expect(jsonRecordBatch.numCols).toEqual(binaryRecordBatch.numCols); for (let i = -1, n = jsonRecordBatch.numCols; ++i < n;) { + (jsonRecordBatch.columns[i] as any).name = jsonRecordBatch.schema.fields[i].name; (expect(jsonRecordBatch.columns[i]) as any).toEqualVector(binaryRecordBatch.columns[i]); } } }); } -function testTableFromBuffersIntegration() { - test(`json and arrow buffers report the same values`, () => { - debugger; +function testTableFromBuffersIntegration(jsonData: any, arrowBuffer: Uint8Array) { + test(`json and arrow tables report the same values`, () => { expect.hasAssertions(); const jsonTable = Table.from(jsonData); - const binaryTable = Table.from(arrowBuffers); + const binaryTable = Table.from(arrowBuffer); expect(jsonTable.length).toEqual(binaryTable.length); expect(jsonTable.numCols).toEqual(binaryTable.numCols); for (let i = -1, n = jsonTable.numCols; ++i < n;) { + (jsonTable.columns[i] as any).name = jsonTable.schema.fields[i].name; (expect(jsonTable.columns[i]) as any).toEqualVector(binaryTable.columns[i]); } });