feat: watch for changed integration test files

README.markdown

@@ -404,11 +404,13 @@ The commands below assume you have **Docker** installed. To use a **local** copy
 **Workflow**
 
 - Modifying the plugin implementation:
+  - The most important logic is found in [src/main.ts](src/main.ts).
   - Run `yarn bin2ts` or `yarn bin2ts:local`.  
     _Since the proto files were not changed, you only need to regenerate the typescript files._
   - Run `yarn test` to verify the typescript files are compatible with the reference implementation, and pass other tests.
 - Updating or adding `.proto` files in the integration directory:
-  - Run `yarn build:test` to regenerate the integration test files.
+  - Run `yarn watch` to automatically regenerate test files when proto files change. 
+    - Or run `yarn build:test` to regenerate all integration test files. 
   - Run `yarn test` to retest.
 
 **Contributing**

integration/codegen.sh

@@ -13,6 +13,10 @@
 #
 #    Updates generated output only for the 'simple' and 'value' integration test.
 #
+# ./codegen.sh simple/simple.bin
+#
+#    Updates generated output for the 'simple' integration test. Used by file watcher.
+#
 # Each integration test can optionally have a `parameters.txt` file that will
 # be used as the ts-proto_opt... args for generating that test's code.
 
@@ -25,27 +29,30 @@ N=5
 
 echo "Generating typescript code for integration tests using ${N} cores..."
 
-dir=.
-if [ -n "${1}" ]; then
-  dir="${@}"
+cd $INTEGRATION_DIR
+
+if [[ $# -eq 0 ]]; then
+  FILTER_PATHS=.
+else
+  FILTER_PATHS="${@}"
 fi
 
-cd $INTEGRATION_DIR
-list=$(find $dir -name "*.bin" -type f | grep -v dump-response.bin)
+# Finds .bin files that match the given filter paths, which can be directories or paths to the .bin files themselves.
+BIN_FILES=$(find $FILTER_PATHS -name "*.bin" -type f | grep -v dump-response.bin)
 
-for file in $list; do
-  echo "${file}"
+for BIN_FILE in $BIN_FILES; do
+  echo "${BIN_FILE}"
   # Strip the longest suffix starting at the 1st slash
-  dir="${file##./}"
-  dir="${dir%%/*}"
-  params=""
+  TEST_DIR="${BIN_FILE##./}"
+  TEST_DIR="${TEST_DIR%%/*}"
+  PARAMS=""
 
-  if [ -f "${dir}/parameters.txt" ]; then
-    params=$(cat "${dir}/parameters.txt")
+  if [ -f "${TEST_DIR}/parameters.txt" ]; then
+    PARAMS=$(cat "${TEST_DIR}/parameters.txt")
   fi
 
   ((i=i%N)); ((i++==0)) && wait
-  "../node_modules/.bin/ts-node" "./codegen.ts" "${dir}" "${file}" "${params}" &
+  "../node_modules/.bin/ts-node" "./codegen.ts" "${TEST_DIR}" "${BIN_FILE}" "${PARAMS}" &
 done
 
 wait

integration/pbjs.sh

@@ -4,59 +4,98 @@
 #
 # Only a handful of integration tests use these so we hand-code it one-off.
 #
+# Usage
+#
+#  ./pbjs.sh [TEST_DIRECTORY | PROTO_FILE, ...]
+#
+# Run for all integration tests:
+#   ./pbjs.sh
+#
+# Run for specific integration tests:
+#   ./pbjs.sh simple value
+#   ./pbjs.sh simple/simple.proto simple-long/simple.proto
+
+INTEGRATION_DIR=$(realpath $(dirname "$BASH_SOURCE"))
+
+if [[ $# -eq 0 ]]; then
+  FILTER_PATHS="$INTEGRATION_DIR"
+else
+  FILTER_PATHS=$(echo "${@}" | xargs realpath)
+fi
+
+cd $INTEGRATION_DIR/../
+set -e
+
+function match() {
+  find $FILTER_PATHS -path "$INTEGRATION_DIR/$1/*.proto" -type f  | grep -q .
+}
 
-# simple/
-yarn pbjs --force-message --force-number -t static-module -o integration/simple/pbjs.js integration/simple/simple.proto
-yarn pbts --no-comments -o integration/simple/pbjs.d.ts integration/simple/pbjs.js
+if match "simple"; then
+  yarn pbjs --force-message --force-number -t static-module -o integration/simple/pbjs.js integration/simple/simple.proto
+  yarn pbts --no-comments -o integration/simple/pbjs.d.ts integration/simple/pbjs.js
+fi
 
-# simple-long
-yarn pbjs --force-message --force-long -t static-module -o integration/simple-long/pbjs.js integration/simple-long/simple.proto
-yarn pbts --no-comments -o integration/simple-long/pbjs.d.ts integration/simple-long/pbjs.js
+if match "simple-long"; then
+  yarn run pbjs --force-message --force-long -t static-module -o integration/simple-long/pbjs.js integration/simple-long/simple.proto
+  yarn run pbts --no-comments -o integration/simple-long/pbjs.d.ts integration/simple-long/pbjs.js
+fi
 
-# simple-long-string
-yarn pbjs --force-message --force-long -t static-module -o integration/simple-long-string/pbjs.js integration/simple-long-string/simple.proto
-yarn pbts --no-comments -o integration/simple-long-string/pbjs.d.ts integration/simple-long-string/pbjs.js
+if match "simple-long-string"; then
+  yarn run pbjs --force-message --force-long -t static-module -o integration/simple-long-string/pbjs.js integration/simple-long-string/simple.proto
+  yarn run pbts --no-comments -o integration/simple-long-string/pbjs.d.ts integration/simple-long-string/pbjs.js
+fi
 
-# vector-tile/
-yarn pbjs --force-message --force-number -t static-module -o integration/vector-tile/pbjs.js integration/vector-tile/vector_tile.proto
-yarn pbts --no-comments -o integration/vector-tile/pbjs.d.ts integration/vector-tile/pbjs.js
+if match "vector-tile"; then
+  yarn run pbjs --force-message --force-number -t static-module -o integration/vector-tile/pbjs.js integration/vector-tile/vector_tile.proto
+  yarn run pbts --no-comments -o integration/vector-tile/pbjs.d.ts integration/vector-tile/pbjs.js
+fi
 
-# nestjs-metadata/
-yarn pbjs --force-message --force-number -t static-module -o integration/nestjs-metadata/pbjs.js integration/nestjs-metadata/hero.proto
-yarn pbts --no-comments -o integration/nestjs-metadata/pbjs.d.ts integration/nestjs-metadata/pbjs.js
+if match "nestjs-metadata"; then
+  yarn run pbjs --force-message --force-number -t static-module -o integration/nestjs-metadata/pbjs.js integration/nestjs-metadata/hero.proto
+  yarn run pbts --no-comments -o integration/nestjs-metadata/pbjs.d.ts integration/nestjs-metadata/pbjs.js
+fi
 
-# nestjs-metadata-observables/
-yarn pbjs --force-message --force-number -t static-module -o integration/nestjs-metadata-observables/pbjs.js integration/nestjs-metadata-observables/hero.proto
-yarn pbts --no-comments -o integration/nestjs-metadata-observables/pbjs.d.ts integration/nestjs-metadata-observables/pbjs.js
+if match "nestjs-metadata-observables"; then
+  yarn run pbjs --force-message --force-number -t static-module -o integration/nestjs-metadata-observables/pbjs.js integration/nestjs-metadata-observables/hero.proto
+  yarn run pbts --no-comments -o integration/nestjs-metadata-observables/pbjs.d.ts integration/nestjs-metadata-observables/pbjs.js
+fi
 
-# nestjs-metadata-restparameters/
-yarn pbjs --force-message --force-number -t static-module -o integration/nestjs-metadata-restparameters/pbjs.js integration/nestjs-metadata-restparameters/hero.proto
-yarn pbts --no-comments -o integration/nestjs-metadata-restparameters/pbjs.d.ts integration/nestjs-metadata-restparameters/pbjs.js
+if match "nestjs-metadata-restparameters"; then
+  yarn run pbjs --force-message --force-number -t static-module -o integration/nestjs-metadata-restparameters/pbjs.js integration/nestjs-metadata-restparameters/hero.proto
+  yarn run pbts --no-comments -o integration/nestjs-metadata-restparameters/pbjs.d.ts integration/nestjs-metadata-restparameters/pbjs.js
+fi
 
-# nestjs-simple/
-yarn pbjs --force-message --force-number -t static-module -o integration/nestjs-simple/pbjs.js integration/nestjs-simple/hero.proto
-yarn pbts --no-comments -o integration/nestjs-simple/pbjs.d.ts integration/nestjs-simple/pbjs.js
+if match "nestjs-simple"; then
+  yarn run pbjs --force-message --force-number -t static-module -o integration/nestjs-simple/pbjs.js integration/nestjs-simple/hero.proto
+  yarn run pbts --no-comments -o integration/nestjs-simple/pbjs.d.ts integration/nestjs-simple/pbjs.js
+fi
 
-# nestjs-simple-observables/
-yarn pbjs --force-message --force-number -t static-module -o integration/nestjs-simple-observables/pbjs.js integration/nestjs-simple-observables/hero.proto
-yarn pbts --no-comments -o integration/nestjs-simple-observables/pbjs.d.ts integration/nestjs-simple-observables/pbjs.js
+if match "nestjs-simple-observables"; then
+  yarn run pbjs --force-message --force-number -t static-module -o integration/nestjs-simple-observables/pbjs.js integration/nestjs-simple-observables/hero.proto
+  yarn run pbts --no-comments -o integration/nestjs-simple-observables/pbjs.d.ts integration/nestjs-simple-observables/pbjs.js
+fi
 
-# nestjs-simple-restparameters/
-yarn pbjs --force-message --force-number -t static-module -o integration/nestjs-simple-restparameters/pbjs.js integration/nestjs-simple-restparameters/hero.proto
-yarn pbts --no-comments -o integration/nestjs-simple-restparameters/pbjs.d.ts integration/nestjs-simple-restparameters/pbjs.js
+if match "nestjs-simple-restparameters"; then
+  yarn run pbjs --force-message --force-number -t static-module -o integration/nestjs-simple-restparameters/pbjs.js integration/nestjs-simple-restparameters/hero.proto
+  yarn run pbts --no-comments -o integration/nestjs-simple-restparameters/pbjs.d.ts integration/nestjs-simple-restparameters/pbjs.js
+fi
 
-# oneof-proprties/
-yarn pbjs --force-message --force-number -t static-module -o integration/oneof-properties/pbjs.js integration/oneof-properties/oneof.proto
-yarn pbts --no-comments -o integration/oneof-properties/pbjs.d.ts integration/oneof-properties/pbjs.js
+if match "oneof-properties"; then
+  yarn run pbjs --force-message --force-number -t static-module -o integration/oneof-properties/pbjs.js integration/oneof-properties/oneof.proto
+  yarn run pbts --no-comments -o integration/oneof-properties/pbjs.d.ts integration/oneof-properties/pbjs.js
+fi
 
-# oneof-unions/
-yarn pbjs --force-message --force-number -t static-module -o integration/oneof-unions/pbjs.js integration/oneof-unions/oneof.proto
-yarn pbts --no-comments -o integration/oneof-unions/pbjs.d.ts integration/oneof-unions/pbjs.js
+if match "oneof-unions"; then
+  yarn run pbjs --force-message --force-number -t static-module -o integration/oneof-unions/pbjs.js integration/oneof-unions/oneof.proto
+  yarn run pbts --no-comments -o integration/oneof-unions/pbjs.d.ts integration/oneof-unions/pbjs.js
+fi
 
-# struct/
-yarn pbjs --force-message --force-number -t static-module -o integration/struct/pbjs.js integration/struct/struct.proto
-yarn pbts --no-comments -o integration/struct/pbjs.d.ts integration/struct/pbjs.js
+if match "struct"; then
+  yarn run pbjs --force-message --force-number -t static-module -o integration/struct/pbjs.js integration/struct/struct.proto
+  yarn run pbts --no-comments -o integration/struct/pbjs.d.ts integration/struct/pbjs.js
+fi
 
-# value/
-yarn pbjs --force-message --force-number -t static-module -o integration/value/pbjs.js integration/value/value.proto
-yarn pbts --no-comments -o integration/value/pbjs.d.ts integration/value/pbjs.js
+if match "value"; then
+  yarn run pbjs --force-message --force-number -t static-module -o integration/value/pbjs.js integration/value/value.proto
+  yarn run pbts --no-comments -o integration/value/pbjs.d.ts integration/value/pbjs.js
+fi

integration/update-bins.sh

@@ -12,7 +12,11 @@ else
   PLUGIN_PATH="protoc-gen-dump"
 fi
 
-PROTO_FILES=$(find . -name "*.proto" -type f)
+if [[ $# -eq 0 ]]; then
+  PROTO_FILES=$(find . -name "*.proto" -type f)
+else
+  PROTO_FILES=$@
+fi
 
 for FILE in $PROTO_FILES; do
   echo "${FILE}"

integration/watch.ts

@@ -0,0 +1,86 @@
+const chokidar = require('chokidar');
+const spawn = require('child_process').spawn;
+
+main()
+
+function main() {
+    const yarn = process.platform === 'win32' ? 'yarn.cmd' : 'yarn'
+
+    if (process.argv.includes('-h') || process.argv.includes('--help')) {
+        console.log(`
+Watches the integration directory for changes and regenerates .bin and .ts files.
+
+Usage:
+    $ yarn watch 
+    $ ts-node watch.ts [options]
+
+Options:
+    -h, --help    Show this help message
+    --polling     Use polling instead of native watchers
+        `)
+        process.exit(0)
+    }
+    const usePolling = process.argv.includes('--polling')
+
+    process.chdir("integration");
+
+    chokidar
+      .watch("*/*.proto", { ignoreInitial: true, usePolling  })
+      .on('all', yarnRunHandler(yarn, 'proto2bin'));
+
+    chokidar
+      .watch("*/*.proto", { ignoreInitial: true, usePolling })
+      .on('all', yarnRunHandler(yarn, 'proto2pbjs'));
+
+    chokidar
+      .watch("*/*.bin", { ignoreInitial: true, usePolling })
+      .on('all', yarnRunHandler(yarn, 'bin2ts'));
+}
+
+const colors = {
+    none: '',
+    reset: '\x1b[0m',
+    green: '\x1b[32m',
+    red: '\x1b[31m',
+    yellow: '\x1b[33m',
+    blue: '\x1b[34m',
+    magenta: '\x1b[35m',
+    cyan: '\x1b[36m'
+}
+
+function yarnRunHandler(yarn: string, task: string) {
+    return (event: "add" | "addDir" | "change" | "unlink" | "unlinkDir", path: string) => {
+        if (event !== 'add' && event !== 'change') {
+            return;
+        }
+
+        path = path.replace(/\\/g, "/"); // windows
+
+        yarnRun(yarn, task, path);
+    }
+}
+
+function yarnRun(yarn: string, task: string, path: string) {
+    const yarnArgs = [task, path];
+
+    console.log(formatLog(colors.green, task, path, `${yarn} ${yarnArgs.join(' ')}`));
+
+    const yarnProcess = spawn(yarn, yarnArgs);
+    yarnProcess.stdout.on('data', (data: Buffer) => console.log(formatLog(colors.none, task, path, data.toString())));
+    yarnProcess.stderr.on('data', (data: Buffer) => console.error(formatLog(colors.red, task, path, data.toString())));
+    yarnProcess.on('error', (err: Error) => console.error(formatLog(colors.red, task, path, err.message)));
+    yarnProcess.on('close', (code: number) => {
+        if (code !== 0) {
+            console.error(formatLog(colors.red, task, path, `Exited with code ${code}`));
+        }
+    });
+}
+
+function formatLog(color: string, task: string, path: string, message: string) {
+    return message
+        .split('\n')
+        .filter(line => line.length)
+        .map(line => `${colors.reset}${path} ${colors.cyan}[${task}]${colors.reset} ${color}` + line)
+        .join('\n')
+      + colors.reset;
+}

package.json

@@ -21,7 +21,8 @@
     "test": "yarn jest -c jest.config.js --maxWorkers=2",
     "prettier": "prettier --write {src,tests}/**/*.ts",
     "prettier:check": "prettier --list-different {src,tests}/**/*.ts",
-    "setup:docker": "docker-compose build"
+    "setup:docker": "docker-compose build",
+    "watch": "ts-node integration/watch.ts"
   },
   "files": [
     "build"
@@ -45,6 +46,7 @@
     "@semantic-release/release-notes-generator": "^9.0.2",
     "@types/jest": "^26.0.22",
     "@types/node": "^14.14.37",
+    "chokidar": "^3.5.2",
     "jest": "^26.6.3",
     "prettier": "^2.2.1",
     "reflect-metadata": "^0.1.13",

src/main.ts

@@ -1009,7 +1009,6 @@ function generateEncode(ctx: Context, fullName: string, messageDesc: DescriptorP
 function generateFromJson(ctx: Context, fullName: string, messageDesc: DescriptorProto): Code {
   const { options, utils, typeMap } = ctx;
   const chunks: Code[] = [];
-  const choiceFieldChunks: Code[] = [];
 
   // create the basic function declaration
   chunks.push(code`