feat: build_manifest default tweaks. (#2287)

If rebuildPatterns are not provided in build_manifest.json, default to the project dir. Can also exclude empty dependencies.
AztecProtocol · Sep 13, 2023 · c8a5cfb · c8a5cfb
1 parent c46b3c8
commit c8a5cfb
Show file tree

Hide file tree

Showing 27 changed files with 57 additions and 252 deletions.
diff --git a/build-system/scripts/query_manifest b/build-system/scripts/query_manifest
@@ -15,6 +15,7 @@ function get_deps {
   local TYPE=$(jq -r ".\"$1\".dependencies | type" $MANIFEST)
   if [ "$TYPE" == "string" ]; then
     # Execute string as command relative to buildDir to retrieve dependencies.
+    local BUILD_DIR=$($0 buildDir $1)
     local CMD=$BUILD_DIR/$(jq -r ".\"$1\".dependencies")
     if [ ! -f "$CMD" ]; then
       >&2 echo "Dependency script not found: $CMD"
@@ -24,10 +25,11 @@ function get_deps {
     DEPS=($($CMD $PROJECT_DIR))
   elif [ "$TYPE" == "null" ]; then
     # Execute default script relative to buildDir to retrieve dependencies.
+    local BUILD_DIR=$($0 buildDir $1)
     local CMD=$BUILD_DIR/scripts/get_dependencies.sh
     if [ ! -f "$CMD" ]; then
-      >&2 echo "Dependency script not found: $CMD"
-      exit 1
+      DEPS=()
+      return
     fi
     local PROJECT_DIR=$($0 projectDir $1)
     DEPS=($($CMD $PROJECT_DIR))
@@ -43,22 +45,25 @@ function add_rebuild_patterns {
   local TYPE=$(jq -r ".\"$1\".rebuildPatterns | type" $MANIFEST)
   if [ "$TYPE" == "string" ]; then
     local FILE=$(jq -r ".\"$1\".rebuildPatterns" $MANIFEST)
-    local BUILD_DIR=$($0 buildDir $1)
-    PATTERNS=(${PATTERNS[@]} $(cat $BUILD_DIR/$FILE))
+    local PROJECT_DIR=$($0 projectDir $1)
+    PATTERNS=(${PATTERNS[@]} $(cat $PROJECT_DIR/$FILE))
   elif [ "$TYPE" == "array" ]; then
     PATTERNS=(${PATTERNS[@]} $(jq -r ".\"$1\".rebuildPatterns | .[]" $MANIFEST))
+  elif [ "$TYPE" == "null" ]; then
+    local PROJECT_DIR=$($0 relativeProjectDir $1)
+    PATTERNS=(${PATTERNS[@]} "^$PROJECT_DIR/")
   else
-    >&2 echo "Missing rebuildPatterns property. Either filename as string, or patterns as array."
+    >&2 echo "rebuildPatterns must be array, string, or null."
     exit 1
   fi
 }
 
 case "$CMD" in
   dockerfile)
-    # In the manifest, the path is relative to buildDir. Return absolute path.
-    BUILD_DIR=$($0 buildDir $REPO)
+    # In the manifest, the path is relative to projectDir. Return absolute path.
+    PROJECT_DIR=$($0 projectDir $REPO)
     DOCKERFILE=$(jq -r ".\"$REPO\".dockerfile // \"Dockerfile\"" $MANIFEST)
-    echo $BUILD_DIR/$DOCKERFILE
+    echo $PROJECT_DIR/$DOCKERFILE
     ;;
   buildDir)
     # In the manifest, the path is relative to the repo root. Return absolute path.
@@ -71,10 +76,15 @@ case "$CMD" in
     echo $ROOT_PATH/$PROJECT_DIR
     ;;
   relativeProjectDir)
+    # Return the relative path as it is in the manifest.
     jq -r ".\"$REPO\".projectDir // .\"$REPO\".buildDir" $MANIFEST
     ;;
   dependencies)
-    BUILD_DIR=$($0 buildDir $REPO)
+    # Get dependencies for a given repo.
+    # If no entry in the manifest file, attempt to call <projectDir>/scripts/get_dependencies.sh if exists, else empty.
+    # If a string, attempt to call <projectDir>/<string> if exists, else error.
+    # If an array, the array lists the dependencies.
+    # Recursively descend "unvisited" dependencies to collect all dependencies.
     declare -A ALL_DEPS
     add_deps() {
       if [[ -v ALL_DEPS[$1] ]]; then
@@ -87,11 +97,18 @@ case "$CMD" in
       done
     }
     add_deps $REPO
+    # Remove ourself as a dependency.
+    unset ALL_DEPS["$REPO"]
     for KEY in "${!ALL_DEPS[@]}"; do
       echo $KEY
     done | sort
     ;;
   rebuildPatterns)
+    # Get rebuild patterns for a given repo (the file patterns that if changed result in rebuilds).
+    # First add rebuild patterns for requested repo, then add rebuild patterns for each dependency.
+    # If no rebuild patterns are given, the result is ["^<projectDir>/"].
+    # If a projects rebuildPattern is a string, the rebuild patterns are in <projectDir>/<string>.
+    # If an array, the array lists the rebuild patterns.
     DEPS=($($0 dependencies $REPO))
     PATTERNS=()
     add_rebuild_patterns $REPO

diff --git a/build_manifest.json b/build_manifest.json
@@ -2,48 +2,40 @@
   "barretenberg-x86_64-linux-clang": {
     "buildDir": "barretenberg/cpp",
     "dockerfile": "dockerfiles/Dockerfile.x86_64-linux-clang",
-    "rebuildPatterns": ".rebuild_patterns",
-    "dependencies": []
+    "rebuildPatterns": ".rebuild_patterns"
   },
   "barretenberg-x86_64-linux-clang-assert": {
     "buildDir": "barretenberg/cpp",
     "dockerfile": "dockerfiles/Dockerfile.x86_64-linux-clang-assert",
-    "rebuildPatterns": ".rebuild_patterns",
-    "dependencies": []
+    "rebuildPatterns": ".rebuild_patterns"
   },
   "barretenberg-x86_64-linux-clang-fuzzing": {
     "buildDir": "barretenberg/cpp",
     "dockerfile": "dockerfiles/Dockerfile.x86_64-linux-clang-fuzzing",
-    "rebuildPatterns": ".rebuild_patterns",
-    "dependencies": []
+    "rebuildPatterns": ".rebuild_patterns"
   },
   "barretenberg-x86_64-linux-gcc": {
     "buildDir": "barretenberg/cpp",
     "dockerfile": "dockerfiles/Dockerfile.x86_64-linux-gcc",
-    "rebuildPatterns": ".rebuild_patterns",
-    "dependencies": []
+    "rebuildPatterns": ".rebuild_patterns"
   },
   "barretenberg-wasm-linux-clang": {
     "buildDir": "barretenberg/cpp",
     "dockerfile": "dockerfiles/Dockerfile.wasm-linux-clang",
-    "rebuildPatterns": ".rebuild_patterns",
-    "dependencies": []
+    "rebuildPatterns": ".rebuild_patterns"
   },
   "bb.js": {
     "buildDir": "barretenberg/ts",
-    "rebuildPatterns": ["^barretenberg/ts/"],
     "dependencies": ["barretenberg-wasm-linux-clang"]
   },
   "barretenberg-acir-tests-bb": {
     "buildDir": "barretenberg/acir_tests",
     "dockerfile": "Dockerfile.bb",
-    "rebuildPatterns": ["^barretenberg/acir_tests/"],
     "dependencies": ["barretenberg-x86_64-linux-clang-assert"]
   },
   "barretenberg-acir-tests-bb.js": {
     "buildDir": "barretenberg/acir_tests",
     "dockerfile": "Dockerfile.bb.js",
-    "rebuildPatterns": ["^barretenberg/acir_tests/"],
     "dependencies": ["bb.js"]
   },
   "circuits-wasm-linux-clang": {
@@ -79,19 +71,14 @@
   "docs": {
     "buildDir": ".",
     "dockerfile": "docs/Dockerfile",
-    "rebuildPatterns": ["^docs/", "^.*.cpp$", "^.*.ts$"],
-    "dependencies": []
+    "rebuildPatterns": ["^docs/", "^.*.cpp$", "^.*.ts$"]
   },
   "l1-contracts": {
-    "buildDir": "l1-contracts",
-    "dockerfile": "Dockerfile",
-    "rebuildPatterns": ["^l1-contracts/"],
-    "dependencies": []
+    "buildDir": "l1-contracts"
   },
   "l1-artifacts": {
     "buildDir": "yarn-project",
     "projectDir": "yarn-project/l1-artifacts",
-    "dockerfile": "l1-artifacts/Dockerfile",
     "rebuildPatterns": ["^l1-contracts/", "^yarn-project/l1-artifacts/"],
     "dependencies": []
   },
@@ -111,92 +98,65 @@
   },
   "acir-simulator": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/acir-simulator",
-    "dockerfile": "acir-simulator/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/acir-simulator/"]
+    "projectDir": "yarn-project/acir-simulator"
   },
   "archiver": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/archiver",
-    "dockerfile": "archiver/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/archiver/"]
+    "projectDir": "yarn-project/archiver"
   },
   "cli": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/cli",
-    "dockerfile": "cli/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/cli/"]
+    "projectDir": "yarn-project/cli"
   },
   "aztec-rpc": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/aztec-rpc",
-    "dockerfile": "aztec-rpc/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/aztec-rpc/"]
+    "projectDir": "yarn-project/aztec-rpc"
   },
   "aztec-sandbox": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/aztec-sandbox",
-    "dockerfile": "aztec-sandbox/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/aztec-sandbox/"]
+    "projectDir": "yarn-project/aztec-sandbox"
   },
   "aztec.js": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/aztec.js",
-    "dockerfile": "aztec.js/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/aztec.js/"]
+    "projectDir": "yarn-project/aztec.js"
   },
   "canary-build": {
     "buildDir": "yarn-project",
     "projectDir": "yarn-project/canary",
-    "dockerfile": "canary/Dockerfile.build",
-    "rebuildPatterns": ["^yarn-project/canary/"]
+    "dockerfile": "Dockerfile.build"
   },
   "canary": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/canary",
-    "dockerfile": "canary/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/canary/"]
+    "projectDir": "yarn-project/canary"
   },
   "circuits.js": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/circuits.js",
-    "dockerfile": "circuits.js/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/circuits.js/"]
+    "projectDir": "yarn-project/circuits.js"
   },
   "end-to-end": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/end-to-end",
-    "dockerfile": "end-to-end/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/end-to-end/"]
+    "projectDir": "yarn-project/end-to-end"
   },
   "ethereum": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/ethereum",
-    "dockerfile": "ethereum/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/ethereum/"]
+    "projectDir": "yarn-project/ethereum"
   },
   "foundation": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/foundation",
-    "dockerfile": "foundation/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/foundation/"]
+    "projectDir": "yarn-project/foundation"
   },
   "key-store": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/key-store",
-    "dockerfile": "key-store/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/key-store/"]
+    "projectDir": "yarn-project/key-store"
   },
   "merkle-tree": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/merkle-tree",
-    "dockerfile": "merkle-tree/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/merkle-tree/"]
+    "projectDir": "yarn-project/merkle-tree"
   },
   "noir-contracts-build": {
     "buildDir": "yarn-project",
     "projectDir": "yarn-project/noir-contracts",
-    "dockerfile": "noir-contracts/Dockerfile.build",
+    "dockerfile": "Dockerfile.build",
     "rebuildPatterns": [
       "^yarn-project/noir-contracts/",
       "^yarn-project/aztec-nr/"
@@ -205,64 +165,46 @@
   "noir-contracts": {
     "buildDir": "yarn-project",
     "projectDir": "yarn-project/noir-contracts",
-    "dockerfile": "noir-contracts/Dockerfile",
     "rebuildPatterns": [
       "^yarn-project/noir-contracts/",
       "^yarn-project/aztec-nr/"
     ]
   },
   "noir-compiler": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/noir-compiler",
-    "dockerfile": "noir-compiler/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/noir-compiler/"]
+    "projectDir": "yarn-project/noir-compiler"
   },
   "p2p": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/p2p",
-    "dockerfile": "p2p/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/p2p/"]
+    "projectDir": "yarn-project/p2p"
   },
   "p2p-bootstrap": {
     "buildDir": "yarn-project",
     "projectDir": "yarn-project/p2p-bootstrap",
-    "dockerfile": "p2p/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/p2p-bootstrap/"]
+    "dockerfile": "../p2p/Dockerfile"
   },
   "prover-client": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/prover-client",
-    "dockerfile": "prover-client/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/prover-client/"]
+    "projectDir": "yarn-project/prover-client"
   },
   "rollup-provider": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/rollup-provider",
-    "dockerfile": "rollup-provider/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/rollup-provider/"]
+    "projectDir": "yarn-project/rollup-provider"
   },
   "aztec-node": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/aztec-node",
-    "dockerfile": "aztec-node/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/aztec-node/"]
+    "projectDir": "yarn-project/aztec-node"
   },
   "sequencer-client": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/sequencer-client",
-    "dockerfile": "sequencer-client/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/sequencer-client/"]
+    "projectDir": "yarn-project/sequencer-client"
   },
   "types": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/types",
-    "dockerfile": "types/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/types/"]
+    "projectDir": "yarn-project/types"
   },
   "world-state": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/world-state",
-    "dockerfile": "world-state/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/world-state/"]
+    "projectDir": "yarn-project/world-state"
   }
 }
diff --git a/yarn-project/acir-simulator/package.json b/yarn-project/acir-simulator/package.json
@@ -11,8 +11,6 @@
     "tsconfig": "./tsconfig.json"
   },
   "scripts": {
-    "prepare": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json",
-    "prepare:check": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json --check",
     "build": "yarn clean && tsc -b",
     "build:dev": "tsc -b --watch",
     "clean": "rm -rf ./dest .tsbuildinfo",

diff --git a/yarn-project/archiver/package.json b/yarn-project/archiver/package.json
@@ -11,8 +11,6 @@
     "tsconfig": "./tsconfig.json"
   },
   "scripts": {
-    "prepare": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json",
-    "prepare:check": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json --check",
     "build": "yarn clean && tsc -b",
     "build:dev": "tsc -b --watch",
     "clean": "rm -rf ./dest .tsbuildinfo",

diff --git a/yarn-project/aztec-node/package.json b/yarn-project/aztec-node/package.json
@@ -12,8 +12,6 @@
     "tsconfig": "./tsconfig.json"
   },
   "scripts": {
-    "prepare": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json",
-    "prepare:check": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json --check",
     "build": "yarn clean && tsc -b",
     "build:dev": "tsc -b --watch",
     "clean": "rm -rf ./dest .tsbuildinfo",

diff --git a/yarn-project/aztec-rpc/package.json b/yarn-project/aztec-rpc/package.json
@@ -11,8 +11,6 @@
     "tsconfig": "./tsconfig.json"
   },
   "scripts": {
-    "prepare": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json",
-    "prepare:check": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json --check",
     "build": "yarn clean && tsc -b",
     "build:dev": "tsc -b --watch",
     "clean": "rm -rf ./dest .tsbuildinfo",

diff --git a/yarn-project/aztec-sandbox/package.json b/yarn-project/aztec-sandbox/package.json
@@ -14,13 +14,11 @@
     "tsconfig": "./tsconfig.json"
   },
   "scripts": {
-    "prepare": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json",
     "build": "yarn clean && tsc -b",
     "start": "node --no-warnings ./dest/bin",
     "clean": "rm -rf ./dest .tsbuildinfo",
     "formatting": "run -T prettier --check ./src && run -T eslint ./src",
     "formatting:fix": "run -T prettier -w ./src",
-    "prepare:check": "node ../yarn-project-base/scripts/update_build_manifest.mjs package.json --check",
     "build:dev": "tsc -b --watch",
     "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --passWithNoTests",
     "run:example:token": "DEBUG='aztec:*' node ./dest/examples/private_token_contract.js",