From 3535d1374bee4663afcd812c373d151ca2251fb5 Mon Sep 17 00:00:00 2001 From: rochdev Date: Fri, 17 Jan 2025 13:14:44 -0500 Subject: [PATCH 1/4] precompile the v8 compile cache when building --- Dockerfile | 3 +++ package.json | 3 ++- src/runtime/module_importer.js | 21 +++++++++++++++++++++ yarn.lock | 5 +++++ 4 files changed, 31 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c70236f7..87d709cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,5 +61,8 @@ RUN find /nodejs/node_modules -name "*.cjs.map" -delete RUN find /nodejs/node_modules -name "*.ts.map" -delete RUN find /nodejs/node_modules -name "*.md" -delete +# Warm up compile cache +RUN node -e "require('/nodejs/node_modules/datadog-lambda-js/runtime/module_importer').initTracer()" + FROM scratch COPY --from=builder /nodejs / diff --git a/package.json b/package.json index e79b57e0..16336279 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,8 @@ "hot-shots": "8.5.0", "promise-retry": "^2.0.1", "serialize-error": "^8.1.0", - "shimmer": "1.2.1" + "shimmer": "1.2.1", + "v8-compile-cache": "^2.4.0" }, "jest": { "verbose": true, diff --git a/src/runtime/module_importer.js b/src/runtime/module_importer.js index 9732e022..276cced2 100644 --- a/src/runtime/module_importer.js +++ b/src/runtime/module_importer.js @@ -1,6 +1,25 @@ const { logDebug, updateDDTags } = require("../utils"); +function compileCache () { + const { FileSystemBlobStore, NativeCompileCache } = require('v8-compile-cache').__TEST__ + + const cacheDir = __dirname + const prefix = 'module_importer' + const blobStore = new FileSystemBlobStore(cacheDir, prefix) + + const nativeCompileCache = new NativeCompileCache() + nativeCompileCache.setCacheStore(blobStore) + nativeCompileCache.install() + + process.once('exit', () => { + if (blobStore.isDirty()) { + blobStore.save() + } + nativeCompileCache.uninstall() + }) +} + // Currently no way to prevent typescript from auto-transpiling import into require, // so we expose a wrapper in js exports.import = function (path) { @@ -8,6 +27,8 @@ exports.import = function (path) { } exports.initTracer = function () { + compileCache() + // Looks for the function local version of dd-trace first, before using // the version provided by the layer const path = require.resolve("dd-trace", { paths: ["/var/task/node_modules", ...module.paths] }); diff --git a/yarn.lock b/yarn.lock index 7794afd9..e8ed34a3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4075,6 +4075,11 @@ uuid@^9.0.1: resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30" integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA== +v8-compile-cache@^2.4.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.4.0.tgz#cdada8bec61e15865f05d097c5f4fd30e94dc128" + integrity sha512-ocyWc3bAHBB/guyqJQVI5o4BZkPhznPYUG2ea80Gond/BgNWpap8TOmLSeeQG7bnh2KMISxskdADG59j7zruhw== + v8-to-istanbul@^8.1.0: version "8.1.1" resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-8.1.1.tgz#77b752fd3975e31bbcef938f85e9bd1c7a8d60ed" From 567df205adc487f9b3f2c1153ad28af9d59f1c4a Mon Sep 17 00:00:00 2001 From: rochdev Date: Fri, 17 Jan 2025 13:37:41 -0500 Subject: [PATCH 2/4] improve comment --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 87d709cb..563232a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,7 +61,7 @@ RUN find /nodejs/node_modules -name "*.cjs.map" -delete RUN find /nodejs/node_modules -name "*.ts.map" -delete RUN find /nodejs/node_modules -name "*.md" -delete -# Warm up compile cache +# Warm up v8 compile cache RUN node -e "require('/nodejs/node_modules/datadog-lambda-js/runtime/module_importer').initTracer()" FROM scratch From eb2bd8b6da28f96789fac6cbd22f36600086d6ad Mon Sep 17 00:00:00 2001 From: rochdev Date: Fri, 17 Jan 2025 15:08:26 -0500 Subject: [PATCH 3/4] fix indentation --- src/runtime/module_importer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/module_importer.js b/src/runtime/module_importer.js index 276cced2..10366231 100644 --- a/src/runtime/module_importer.js +++ b/src/runtime/module_importer.js @@ -14,7 +14,7 @@ function compileCache () { process.once('exit', () => { if (blobStore.isDirty()) { - blobStore.save() + blobStore.save() } nativeCompileCache.uninstall() }) From e2875aba953f7495d25bc14c8b37d0e098849ebd Mon Sep 17 00:00:00 2001 From: rochdev Date: Fri, 17 Jan 2025 16:06:37 -0500 Subject: [PATCH 4/4] fix path --- Dockerfile | 74 +++++++++++++++++++++++++++--------------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/Dockerfile b/Dockerfile index 563232a8..b8d47283 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM $image as builder ARG image # Create the directory structure required for AWS Lambda Layer -RUN mkdir -p /nodejs/node_modules/ +RUN mkdir -p /opt/nodejs/node_modules/ # Install dev dependencies COPY . datadog-lambda-js @@ -12,10 +12,10 @@ RUN yarn install # Build the lambda layer RUN yarn build -RUN cp -r dist /nodejs/node_modules/datadog-lambda-js -RUN cp ./src/runtime/module_importer.js /nodejs/node_modules/datadog-lambda-js/runtime +RUN cp -r dist /opt/nodejs/node_modules/datadog-lambda-js +RUN cp ./src/runtime/module_importer.js /opt/nodejs/node_modules/datadog-lambda-js/runtime -RUN cp ./src/handler.mjs /nodejs/node_modules/datadog-lambda-js +RUN cp ./src/handler.mjs /opt/nodejs/node_modules/datadog-lambda-js RUN rm -rf node_modules # Move dd-trace from devDependencies to production dependencies @@ -25,44 +25,44 @@ RUN mv package-new.json package.json # Install dependencies RUN yarn install --production=true # Copy the dependencies to the modules folder -RUN cp -rf node_modules/* /nodejs/node_modules +RUN cp -rf node_modules/* /opt/nodejs/node_modules # Remove the AWS SDK, which is installed in the lambda by default -RUN rm -rf /nodejs/node_modules/aws-sdk -RUN rm -rf /nodejs/node_modules/aws-xray-sdk-core/node_modules/aws-sdk +RUN rm -rf /opt/nodejs/node_modules/aws-sdk +RUN rm -rf /opt/nodejs/node_modules/aws-xray-sdk-core/node_modules/aws-sdk # Remove heavy files from dd-trace which aren't used in a lambda environment -RUN rm -rf /nodejs/node_modules/dd-trace/prebuilds -RUN rm -rf /nodejs/node_modules/dd-trace/dist -RUN rm -rf /nodejs/node_modules/@datadog/libdatadog -RUN rm -rf /nodejs/node_modules/@datadog/native-appsec -RUN rm -rf /nodejs/node_modules/@datadog/native-metrics -RUN rm -rf /nodejs/node_modules/hdr-histogram-js/build -RUN rm -rf /nodejs/node_modules/protobufjs/dist -RUN rm -rf /nodejs/node_modules/protobufjs/cli -RUN rm -rf /nodejs/node_modules/@datadog/pprof/prebuilds/linux-arm -RUN rm -rf /nodejs/node_modules/@datadog/pprof/prebuilds/darwin-arm64 -RUN rm -rf /nodejs/node_modules/@datadog/pprof/prebuilds/darwin-x64 -RUN rm -rf /nodejs/node_modules/@datadog/pprof/prebuilds/win32-ia32 -RUN rm -rf /nodejs/node_modules/@datadog/pprof/prebuilds/win32-x64 -RUN rm -rf /nodejs/node_modules/@datadog/native-iast-taint-tracking -RUN rm -rf /nodejs/node_modules/@datadog/native-iast-rewriter -RUN rm -rf /nodejs/node_modules/@datadog/pprof/prebuilds/linuxmusl-x64 -RUN rm -rf /nodejs/node_modules/jsonpath-plus/src/jsonpath.d.ts -RUN rm -rf /nodejs/node_modules/jsonpath-plus/src/jsonpath-browser.js -RUN rm -rf /nodejs/node_modules/jsonpath-plus/src/dist/index-browser-umd.min.cjs -RUN rm -rf /nodejs/node_modules/jsonpath-plus/src/dist/index-browser-umd.cjs -RUN rm -rf /nodejs/node_modules/jsonpath-plus/src/dist/index-browser-esm.min.js -RUN rm -rf /nodejs/node_modules/jsonpath-plus/src/dist/index-browser-esm.js -RUN find /nodejs/node_modules -name "*.d.ts" -delete -RUN find /nodejs/node_modules -name "*.js.map" -delete -RUN find /nodejs/node_modules -name "*.mjs.map" -delete -RUN find /nodejs/node_modules -name "*.cjs.map" -delete -RUN find /nodejs/node_modules -name "*.ts.map" -delete -RUN find /nodejs/node_modules -name "*.md" -delete +RUN rm -rf /opt/nodejs/node_modules/dd-trace/prebuilds +RUN rm -rf /opt/nodejs/node_modules/dd-trace/dist +RUN rm -rf /opt/nodejs/node_modules/@datadog/libdatadog +RUN rm -rf /opt/nodejs/node_modules/@datadog/native-appsec +RUN rm -rf /opt/nodejs/node_modules/@datadog/native-metrics +RUN rm -rf /opt/nodejs/node_modules/hdr-histogram-js/build +RUN rm -rf /opt/nodejs/node_modules/protobufjs/dist +RUN rm -rf /opt/nodejs/node_modules/protobufjs/cli +RUN rm -rf /opt/nodejs/node_modules/@datadog/pprof/prebuilds/linux-arm +RUN rm -rf /opt/nodejs/node_modules/@datadog/pprof/prebuilds/darwin-arm64 +RUN rm -rf /opt/nodejs/node_modules/@datadog/pprof/prebuilds/darwin-x64 +RUN rm -rf /opt/nodejs/node_modules/@datadog/pprof/prebuilds/win32-ia32 +RUN rm -rf /opt/nodejs/node_modules/@datadog/pprof/prebuilds/win32-x64 +RUN rm -rf /opt/nodejs/node_modules/@datadog/native-iast-taint-tracking +RUN rm -rf /opt/nodejs/node_modules/@datadog/native-iast-rewriter +RUN rm -rf /opt/nodejs/node_modules/@datadog/pprof/prebuilds/linuxmusl-x64 +RUN rm -rf /opt/nodejs/node_modules/jsonpath-plus/src/jsonpath.d.ts +RUN rm -rf /opt/nodejs/node_modules/jsonpath-plus/src/jsonpath-browser.js +RUN rm -rf /opt/nodejs/node_modules/jsonpath-plus/src/dist/index-browser-umd.min.cjs +RUN rm -rf /opt/nodejs/node_modules/jsonpath-plus/src/dist/index-browser-umd.cjs +RUN rm -rf /opt/nodejs/node_modules/jsonpath-plus/src/dist/index-browser-esm.min.js +RUN rm -rf /opt/nodejs/node_modules/jsonpath-plus/src/dist/index-browser-esm.js +RUN find /opt/nodejs/node_modules -name "*.d.ts" -delete +RUN find /opt/nodejs/node_modules -name "*.js.map" -delete +RUN find /opt/nodejs/node_modules -name "*.mjs.map" -delete +RUN find /opt/nodejs/node_modules -name "*.cjs.map" -delete +RUN find /opt/nodejs/node_modules -name "*.ts.map" -delete +RUN find /opt/nodejs/node_modules -name "*.md" -delete # Warm up v8 compile cache -RUN node -e "require('/nodejs/node_modules/datadog-lambda-js/runtime/module_importer').initTracer()" +RUN node -e "require('/opt/nodejs/node_modules/datadog-lambda-js/runtime/module_importer').initTracer()" FROM scratch -COPY --from=builder /nodejs / +COPY --from=builder /opt/nodejs /