Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: Run codegen with N=5, fix update-bins #418

Merged
merged 4 commits into from
Nov 27, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nvmrc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
16.4.0
6 changes: 0 additions & 6 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,5 @@ services:
build:
context: .
dockerfile: "protoc.Dockerfile"
args:
BASE: "node:17-alpine3.14"
BIN: "/ts-proto"
PROTOC_VERSION: "3.19.1"
working_dir: "/host"
volumes:
- "${PWD:-.}:/host"
- ".:/ts-proto"
9 changes: 6 additions & 3 deletions integration/codegen.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@

INTEGRATION_DIR=$(realpath $(dirname "$BASH_SOURCE"))

# Run the code generator in parallel, with one process per core.
N=$(nproc)
# Run the code generator in parallel. Note this is purposefully pinned to 5 because
# CI only has 2 cores, but we can go faster than that, and for me locally using all
# 16 cores is overly taxes the machine/kicks on fans/etc. 5 is a good balance.
N=5

echo "Generating typescript code for integration tests using ${N} cores..."

Expand All @@ -28,6 +30,7 @@ if [ -n "${1}" ]; then
dir="${@}"
fi

cd $INTEGRATION_DIR
list=$(find $dir -name "*.bin" -type f | grep -v dump-response.bin)

for file in $list; do
Expand All @@ -42,7 +45,7 @@ for file in $list; do
fi

((i=i%N)); ((i++==0)) && wait
"${INTEGRATION_DIR}/../node_modules/.bin/ts-node" "${INTEGRATION_DIR}/codegen.ts" "${dir}" "${file}" "${params}" &
"../node_modules/.bin/ts-node" "./codegen.ts" "${dir}" "${file}" "${params}" &
done

wait
Binary file modified integration/grpc-js/google/protobuf/wrappers.bin
Binary file not shown.
Binary file modified integration/meta-typings/google/protobuf/wrappers.bin
Binary file not shown.
Binary file modified integration/meta-typings/google/type/date.bin
Binary file not shown.
Binary file modified integration/meta-typings/import_dir/thing.bin
Binary file not shown.
Binary file modified integration/simple-long-string/google/protobuf/wrappers.bin
Binary file not shown.
Binary file modified integration/simple-long/google/protobuf/wrappers.bin
Binary file not shown.
Binary file modified integration/simple-optionals/google/protobuf/wrappers.bin
Binary file not shown.
Binary file modified integration/simple-optionals/import_dir/thing.bin
Binary file not shown.
Binary file modified integration/simple-snake/google/protobuf/wrappers.bin
Binary file not shown.
Binary file modified integration/simple-snake/import_dir/thing.bin
Binary file not shown.
Binary file not shown.
Binary file modified integration/simple-unrecognized-enum/import_dir/thing.bin
Binary file not shown.
Binary file modified integration/simple/google/protobuf/wrappers.bin
Binary file not shown.
Binary file modified integration/simple/google/type/date.bin
Binary file not shown.
Binary file modified integration/simple/import_dir/thing.bin
Binary file not shown.
Binary file modified integration/type-registry/bar/bar.bin
Binary file not shown.
11 changes: 10 additions & 1 deletion integration/type-registry/google/protobuf/timestamp.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,16 @@ export const protobufPackage = 'google.protobuf';
* .setNanos((int) ((millis % 1000) * 1000000)).build();
*
*
* Example 5: Compute Timestamp from current time in Python.
* Example 5: Compute Timestamp from Java `Instant.now()`.
*
* Instant now = Instant.now();
*
* Timestamp timestamp =
* Timestamp.newBuilder().setSeconds(now.getEpochSecond())
* .setNanos(now.getNano()).build();
*
*
* Example 6: Compute Timestamp from current time in Python.
*
* timestamp = Timestamp()
* timestamp.GetCurrentTime()
Expand Down
6 changes: 4 additions & 2 deletions integration/update-bins.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@ else
PLUGIN_PATH="protoc-gen-dump"
fi

PROTO_FILES=$(find . -maxdepth 2 -name "*.proto" -type f)
PROTO_FILES=$(find . -name "*.proto" -type f)

for FILE in $PROTO_FILES; do
echo "${FILE}"
INPUT_DIR="$(dirname "$FILE")"
# Strip the longest suffix starting at the 1st slash
INPUT_DIR="${FILE##./}"
INPUT_DIR="${INPUT_DIR%%/*}"
OUTPUT_FILE="${FILE%proto}bin"
protoc --experimental_allow_proto3_optional "--plugin=$PLUGIN_PATH" --dump_out=. "${FILE}" "-I${INPUT_DIR}"
mv file.bin "${OUTPUT_FILE}"
Expand Down
8 changes: 4 additions & 4 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@
"build": "yarn tsc",
"build:test": "yarn proto2bin && yarn proto2pbjs && yarn bin2ts",
"build:test:local": "yarn proto2bin:local && yarn proto2pbjs:local && yarn bin2ts:local",
"proto2bin": "docker-compose run --rm --entrypoint bash -w /ts-proto/integration protoc update-bins.sh",
"proto2bin-node": "docker-compose run --rm --entrypoint bash -w /ts-proto/integration node update-bins.sh",
"proto2pbjs": "docker-compose run --rm --entrypoint bash -w /ts-proto/integration protoc pbjs.sh",
"bin2ts": "docker-compose run --rm --entrypoint bash -w /ts-proto/integration protoc codegen.sh",
"proto2bin": "docker-compose run --rm protoc update-bins.sh",
"proto2bin-node": "docker-compose run --rm node update-bins.sh",
"proto2pbjs": "docker-compose run --rm protoc pbjs.sh",
"bin2ts": "docker-compose run --rm protoc codegen.sh",
"proto2bin:local": "integration/update-bins.sh",
"proto2pbjs:local": "integration/pbjs.sh",
"bin2ts:local": "integration/codegen.sh",
Expand Down
9 changes: 3 additions & 6 deletions protoc.Dockerfile
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
# Docker image for protoc
ARG BASE="alpine:3.14.3"
FROM $BASE
FROM node:17-alpine3.14
ARG PROTOC_VERSION="3.19.1"
ARG BIN=""

RUN apk add bash
RUN apk add gcompat
Expand All @@ -11,7 +9,6 @@ RUN mkdir /usr/local/lib/protoc && unzip protoc.zip -d /usr/local/lib/protoc &&
RUN ln -s /usr/local/lib/protoc/bin/protoc /usr/local/bin/protoc

RUN protoc --version
WORKDIR /ts-proto/integration

ENV PATH "$BIN:$PATH"

ENTRYPOINT ["/usr/local/bin/protoc"]
ENTRYPOINT ["/bin/bash"]