diff --git a/.gitignore b/.gitignore index 579946fb0c..96338d2d73 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ local.log /dist-docker /pysrc binary-releases +ts-cli-binaries +prepack tmp .DS_Store !/test/fixtures/**/package-lock.json diff --git a/Makefile b/Makefile index a77f0c9409..5e87120d9e 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,10 @@ # PKG := npx pkg ./ --compress Brotli +BINARY_WRAPPER_DIR = ts-binary-wrapper +EXTENSIBLE_CLI_DIR = cliv2 +BINARY_RELEASES_FOLDER_TS_CLI = binary-releases +BINARY_OUTPUT_FOLDER = binary-releases .DEFAULT: help .PHONY: help @@ -15,11 +19,16 @@ help: @echo 'This Makefile is currently only for building release artifacts.' @echo 'Use `npm run` for CLIv1 scripts.' -binary-releases: - mkdir binary-releases +$(BINARY_RELEASES_FOLDER_TS_CLI): + @mkdir $(BINARY_RELEASES_FOLDER_TS_CLI) -binary-releases/version: | binary-releases - ./release-scripts/next-version.sh > binary-releases/version +$(BINARY_RELEASES_FOLDER_TS_CLI)/version: | $(BINARY_RELEASES_FOLDER_TS_CLI) + ./release-scripts/next-version.sh > $(BINARY_RELEASES_FOLDER_TS_CLI)/version + +ifneq ($(BINARY_OUTPUT_FOLDER), $(BINARY_RELEASES_FOLDER_TS_CLI)) +$(BINARY_OUTPUT_FOLDER)/version: $(BINARY_RELEASES_FOLDER_TS_CLI)/version + @cp $(BINARY_RELEASES_FOLDER_TS_CLI)/version $(BINARY_OUTPUT_FOLDER)/version +endif # prepack is not a typical target. # It modifies package.json files rather than only creating new files. @@ -29,9 +38,9 @@ binary-releases/version: | binary-releases # Only removing "prepack" is not enough. We need to do additional cleanup (see clean-prepack). .INTERMEDIATE: prepack .SECONDARY: prepack -prepack: binary-releases/version +prepack: $(BINARY_RELEASES_FOLDER_TS_CLI)/version @echo "'make prepack' was run. Run 'make clean-prepack' to rollback your package.json changes and this file." > prepack - npm version "$(shell cat binary-releases/version)" --no-git-tag-version --workspaces --include-workspace-root + npm version "$(shell cat $(BINARY_RELEASES_FOLDER_TS_CLI)/version)" --no-git-tag-version --workspaces --include-workspace-root npx ts-node ./release-scripts/prune-dependencies-in-packagejson.ts .PHONY: clean-prepack @@ -39,75 +48,115 @@ clean-prepack: git checkout package.json package-lock.json packages/*/package.json packages/*/package-lock.json rm -f prepack -.PHONY: clean -clean: clean-prepack +.PHONY: clean-ts +clean-ts: npm run clean - rm -f -r binary-releases + rm -f -r $(BINARY_RELEASES_FOLDER_TS_CLI) - -binary-releases/sha256sums.txt.asc: $(wildcard binary-releases/*.sha256) +$(BINARY_OUTPUT_FOLDER)/sha256sums.txt.asc: $(wildcard $(BINARY_OUTPUT_FOLDER)/*.sha256) ./release-scripts/sha256sums.txt.asc.sh -binary-releases/release.json: binary-releases/version $(wildcard binary-releases/*.sha256) +$(BINARY_OUTPUT_FOLDER)/release.json: $(BINARY_OUTPUT_FOLDER)/version $(wildcard $(BINARY_OUTPUT_FOLDER)/*.sha256) ./release-scripts/release.json.sh # --commit-path is forwarded to `git log `. # We're using this to remove CLIv2 changes in v1's changelogs. # :(exclude) syntax: https://git-scm.com/docs/gitglossary.html#Documentation/gitglossary.txt-exclude # Release notes uses version from package.json so we need to prepack beforehand. -binary-releases/RELEASE_NOTES.md: prepack | binary-releases - npx conventional-changelog-cli -p angular -l -r 1 --commit-path ':(exclude)cliv2' > binary-releases/RELEASE_NOTES.md +$(BINARY_OUTPUT_FOLDER)/RELEASE_NOTES.md: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI) + npx conventional-changelog-cli -p angular -l -r 1 --commit-path ':(exclude)cliv2' > $(BINARY_OUTPUT_FOLDER)/RELEASE_NOTES.md # Generates a shasum of a target with the same name. # See "Automatic Variables" in GNU Make docs (linked at the top) %.sha256: % cd $(@D); shasum -a 256 $( $(@F); shasum -a 256 -c $(@F) -binary-releases/snyk.tgz: prepack | binary-releases - mv $(shell npm pack) binary-releases/snyk.tgz - $(MAKE) binary-releases/snyk.tgz.sha256 +$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk.tgz: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI) + mv $(shell npm pack) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk.tgz + $(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk.tgz.sha256 -binary-releases/snyk-fix.tgz: prepack | binary-releases - mv $(shell npm pack --workspace '@snyk/fix') binary-releases/snyk-fix.tgz - $(MAKE) binary-releases/snyk-fix.tgz.sha256 +$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-fix.tgz: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI) + mv $(shell npm pack --workspace '@snyk/fix') $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-fix.tgz + $(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-fix.tgz.sha256 -binary-releases/snyk-protect.tgz: prepack | binary-releases - mv $(shell npm pack --workspace '@snyk/protect') binary-releases/snyk-protect.tgz - $(MAKE) binary-releases/snyk-protect.tgz.sha256 +$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-protect.tgz: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI) + mv $(shell npm pack --workspace '@snyk/protect') $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-protect.tgz + $(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-protect.tgz.sha256 -binary-releases/snyk-alpine: prepack | binary-releases - $(PKG) -t node16-alpine-x64 -o binary-releases/snyk-alpine - $(MAKE) binary-releases/snyk-alpine.sha256 +$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-alpine: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI) + $(PKG) -t node16-alpine-x64 -o $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-alpine + $(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-alpine.sha256 -binary-releases/snyk-linux: prepack | binary-releases - $(PKG) -t node16-linux-x64 -o binary-releases/snyk-linux - $(MAKE) binary-releases/snyk-linux.sha256 +$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-linux: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI) + $(PKG) -t node16-linux-x64 -o $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-linux + $(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-linux.sha256 # Why `--no-bytecode` for Linux/arm64: # arm64 bytecode generation requires various build tools on an x64 build # environment. So disabling until we can support it. It's an optimisation. # https://github.com/vercel/pkg#targets -binary-releases/snyk-linux-arm64: prepack | binary-releases - $(PKG) -t node16-linux-arm64 -o binary-releases/snyk-linux-arm64 --no-bytecode - $(MAKE) binary-releases/snyk-linux-arm64.sha256 +$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-linux-arm64: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI) + $(PKG) -t node16-linux-arm64 -o $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-linux-arm64 --no-bytecode + $(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-linux-arm64.sha256 -binary-releases/snyk-macos: prepack | binary-releases - $(PKG) -t node16-macos-x64 -o binary-releases/snyk-macos - $(MAKE) binary-releases/snyk-macos.sha256 +$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-macos: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI) + $(PKG) -t node16-macos-x64 -o $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-macos + $(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-macos.sha256 -binary-releases/snyk-win.exe: prepack | binary-releases - $(PKG) -t node16-win-x64 -o binary-releases/snyk-win.exe - ./cliv2/scripts/sign_windows.sh binary-releases snyk-win.exe - $(MAKE) binary-releases/snyk-win.exe.sha256 +$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-win.exe: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI) + $(PKG) -t node16-win-x64 -o $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-win.exe + ./cliv2/scripts/sign_windows.sh $(BINARY_RELEASES_FOLDER_TS_CLI) snyk-win.exe + $(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-win.exe.sha256 -binary-releases/snyk-for-docker-desktop-darwin-x64.tar.gz: prepack | binary-releases +$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-for-docker-desktop-darwin-x64.tar.gz: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI) ./docker-desktop/build.sh darwin x64 - $(MAKE) binary-releases/snyk-for-docker-desktop-darwin-x64.tar.gz.sha256 + $(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-for-docker-desktop-darwin-x64.tar.gz.sha256 -binary-releases/snyk-for-docker-desktop-darwin-arm64.tar.gz: prepack | binary-releases +$(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-for-docker-desktop-darwin-arm64.tar.gz: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI) ./docker-desktop/build.sh darwin arm64 - $(MAKE) binary-releases/snyk-for-docker-desktop-darwin-arm64.tar.gz.sha256 + $(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/snyk-for-docker-desktop-darwin-arm64.tar.gz.sha256 -binary-releases/docker-mac-signed-bundle.tar.gz: prepack | binary-releases +$(BINARY_RELEASES_FOLDER_TS_CLI)/docker-mac-signed-bundle.tar.gz: prepack | $(BINARY_RELEASES_FOLDER_TS_CLI) ./release-scripts/docker-desktop-release.sh - $(MAKE) binary-releases/docker-mac-signed-bundle.tar.gz.sha256 + $(MAKE) $(BINARY_RELEASES_FOLDER_TS_CLI)/docker-mac-signed-bundle.tar.gz.sha256 + +# targets responsible for the Wrapper CLI (TS around Golang) +$(BINARY_WRAPPER_DIR)/src/generated: + @mkdir $(BINARY_WRAPPER_DIR)/src/generated/ + +$(BINARY_WRAPPER_DIR)/src/generated/version: $(BINARY_WRAPPER_DIR)/src/generated $(BINARY_RELEASES_FOLDER_TS_CLI)/version + @cp $(BINARY_RELEASES_FOLDER_TS_CLI)/version $(BINARY_WRAPPER_DIR)/src/generated/version + +$(BINARY_WRAPPER_DIR)/src/generated/sha256sums.txt: + @echo "-- Generating $(@F)" + @cat $(BINARY_OUTPUT_FOLDER)/*.sha256 > $(BINARY_WRAPPER_DIR)/src/generated/sha256sums.txt + +.PHONY: build-binary-wrapper +build-binary-wrapper: $(BINARY_WRAPPER_DIR)/src/generated/version $(BINARY_WRAPPER_DIR)/src/generated/sha256sums.txt + @echo "-- Building Typescript Binary Wrapper ($(BINARY_WRAPPER_DIR)/dist/)" + @cd $(BINARY_WRAPPER_DIR) && npm run build + +.PHONY: clean-binary-wrapper +clean-binary-wrapper: + @cd $(BINARY_WRAPPER_DIR) && npm run clean + +.PHONY: pack-binary-wrapper +pack-binary-wrapper: build-binary-wrapper + @echo "-- Packaging tarball ($(BINARY_OUTPUT_FOLDER)/snyk.tgz)" + @mv $(BINARY_WRAPPER_DIR)/$(shell cd $(BINARY_WRAPPER_DIR) && npm pack) $(BINARY_OUTPUT_FOLDER)/snyk.tgz + +.PHONY: test-binary-wrapper +test-binary-wrapper: + @echo "-- Testing binary wrapper" + @cd $(BINARY_WRAPPER_DIR) && npm run test + + +# targets responsible for the complete CLI build +.PHONY: build +build: + @cd $(EXTENSIBLE_CLI_DIR) && $(MAKE) build-full install bindir=$(CURDIR)/$(BINARY_OUTPUT_FOLDER) USE_LEGACY_EXECUTABLE_NAME=1 + +.PHONY: clean +clean: + @cd $(EXTENSIBLE_CLI_DIR) && $(MAKE) clean-full + $(MAKE) clean-prepack diff --git a/cliv2/Makefile b/cliv2/Makefile index 4cc621d217..75b8c31931 100644 --- a/cliv2/Makefile +++ b/cliv2/Makefile @@ -10,6 +10,7 @@ HASH_ALGORITHM = 256 CLI_V2_VERSION_TAG = CLI_V1_VERSION_TAG = CLI_V1_LOCATION = +USE_LEGACY_EXECUTABLE_NAME = # Make directories per convention prefix = /usr/local @@ -70,6 +71,11 @@ APPLICATION_NAME = snyk TEST_NAME = $(APPLICATION_NAME)$(_SEPARATOR)tests V2_PLATFORM_STRING = $(GOOS)$(_SEPARATOR)$(GOARCH) V2_EXECUTABLE_NAME = $(APPLICATION_NAME)$(_SEPARATOR)$(V2_PLATFORM_STRING)$(_EXE_POSTFIX) + +ifneq ($(USE_LEGACY_EXECUTABLE_NAME), $(_EMPTY)) + V2_EXECUTABLE_NAME = $(V1_EXECUTABLE_NAME) +endif + V1_EXECUTABLE_NAME = $(APPLICATION_NAME)-$(V1_PLATFORM_STING)$(_EXE_POSTFIX) V2_DIRECTORY = $(WORKING_DIR)/internal/cliv2 V1_DIRECTORY = $(WORKING_DIR)/internal/embedded/cliv1 @@ -78,6 +84,7 @@ V1_EMBEDDED_FILE_TEMPLATE = $(V1_DIRECTORY)/embedded_binary_template.txt V1_EMBEDDED_FILE_OUTPUT = embedded$(_SEPARATOR)$(V2_PLATFORM_STRING).go V1_WORKING_DIR = $(WORKING_DIR)/.. V1_BUILD_TYPE = build +V1_BINARY_FOLDER = ts-cli-binaries HASH_STRING = $(HASH)$(HASH_ALGORITHM) TEST_SNYK_EXECUTABLE_PATH=$(BUILD_DIR)/$(V2_EXECUTABLE_NAME) TEST_EXECUTABLE_NAME = $(TEST_NAME)$(_SEPARATOR)$(V2_PLATFORM_STRING)$(_EXE_POSTFIX) @@ -85,6 +92,7 @@ SIGN_SCRIPT = sign_$(_GO_OS).sh ISSIGNED_SCRIPT = issigned_$(_GO_OS).sh EMBEDDED_DATA_DIR = $(WORKING_DIR)/internal/embedded/_data + # some make file variables LOG_PREFIX = -- @@ -225,25 +233,25 @@ sign: _cleanup_sha_v2 $(SIGN_SCRIPT) $(BUILD_DIR)/$(V2_EXECUTABLE_NAME).$(HASH_S test-signature: $(ISSIGNED_SCRIPT) # Typescript CLI targets -$(V1_WORKING_DIR)/binary-releases/$(V1_EXECUTABLE_NAME): +$(V1_WORKING_DIR)/$(V1_BINARY_FOLDER)/$(V1_EXECUTABLE_NAME): @echo "$(LOG_PREFIX) Building legacy CLI" @cd $(V1_WORKING_DIR) && npm ci && npm run $(V1_BUILD_TYPE) - @$(MAKE) -C $(V1_WORKING_DIR) binary-releases/$(V1_EXECUTABLE_NAME) + @$(MAKE) -C $(V1_WORKING_DIR) $(V1_BINARY_FOLDER)/$(V1_EXECUTABLE_NAME) BINARY_RELEASES_FOLDER_TS_CLI=$(V1_BINARY_FOLDER) .PHONY: build-ts-cli -build-ts-cli: $(V1_WORKING_DIR)/binary-releases/$(V1_EXECUTABLE_NAME) - $(eval CLI_V1_VERSION_TAG := $(shell cat $(V1_WORKING_DIR)/binary-releases/version)) - $(eval CLI_V1_LOCATION := $(V1_WORKING_DIR)/binary-releases/) +build-ts-cli: $(V1_WORKING_DIR)/$(V1_BINARY_FOLDER)/$(V1_EXECUTABLE_NAME) + $(eval CLI_V1_VERSION_TAG := $(shell cat $(V1_WORKING_DIR)/$(V1_BINARY_FOLDER)/version)) + $(eval CLI_V1_LOCATION := $(V1_WORKING_DIR)/$(V1_BINARY_FOLDER)/) .PHONY: clean-ts-cli clean-ts-cli: @echo "$(LOG_PREFIX) Cleaning legacy CLI" - @$(MAKE) -C $(V1_WORKING_DIR) clean + @$(MAKE) -C $(V1_WORKING_DIR) clean-ts BINARY_RELEASES_FOLDER_TS_CLI=$(V1_BINARY_FOLDER) # build the full CLI (Typescript+Golang) .PHONY: build-full build-full: | build-ts-cli - @$(MAKE) build CLI_V1_VERSION_TAG=$(CLI_V1_VERSION_TAG) CLI_V1_LOCATION="$(CLI_V1_LOCATION)" + @$(MAKE) build build-test CLI_V1_VERSION_TAG=$(CLI_V1_VERSION_TAG) CLI_V1_LOCATION="$(CLI_V1_LOCATION)" # clean the full CLI (Typescript+Golang) .PHONY: clean-full diff --git a/ts-binary-wrapper/jest.config.js b/ts-binary-wrapper/jest.config.js new file mode 100644 index 0000000000..ea6de23b04 --- /dev/null +++ b/ts-binary-wrapper/jest.config.js @@ -0,0 +1,3 @@ +const { createJestConfig } = require('../test/createJestConfig'); + +module.exports = createJestConfig({ displayName: 'ts-binary-wrapper' }); diff --git a/ts-binary-wrapper/package-lock.json b/ts-binary-wrapper/package-lock.json new file mode 100644 index 0000000000..f3824c185a --- /dev/null +++ b/ts-binary-wrapper/package-lock.json @@ -0,0 +1,20 @@ +{ + "name": "ts-binary-wrapper", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "ts-binary-wrapper", + "version": "1.0.0", + "hasInstallScript": true, + "license": "Apache-2.0", + "bin": { + "snyk": "dist/index.js" + }, + "engines": { + "node": ">=16" + } + } + } +} diff --git a/ts-binary-wrapper/package.json b/ts-binary-wrapper/package.json new file mode 100644 index 0000000000..bb35f25bdd --- /dev/null +++ b/ts-binary-wrapper/package.json @@ -0,0 +1,39 @@ +{ + "name": "ts-binary-wrapper", + "version": "1.0.0", + "description": "Wrapper for Snyk's Golang based Extensible CLI.", + "main": "dist/index.js", + "directories": { + "lib": "src", + "test": "test" + }, + "bin": { + "snyk": "dist/index.js" + }, + "engines": { + "node": ">=16" + }, + "scripts": { + "clean": "npx rimraf dist tsconfig.tsbuildinfo src/generated", + "build": "tsc && cp -R src/generated dist/", + "test": "npx jest test/*", + "postinstall": "node dist/bootstrap.js exec" + }, + "keywords": [ + "security", + "vulnerabilities", + "advisories", + "audit", + "snyk", + "scan", + "docker", + "container", + "scanning" + ], + "author": "snyk.io", + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/snyk/snyk.git" + } +} diff --git a/ts-binary-wrapper/src/bootstrap.ts b/ts-binary-wrapper/src/bootstrap.ts new file mode 100644 index 0000000000..eb3b3b8240 --- /dev/null +++ b/ts-binary-wrapper/src/bootstrap.ts @@ -0,0 +1,18 @@ +import * as common from './common'; +import * as process from 'process'; + +const config = common.getCurrentConfiguration(); +export const executable = config.getLocalLocation(); + +if (process.argv.includes('exec')) { + const filenameShasum = config.getShasumFile(); + const downloadUrl = config.getDownloadLocation(); + + common + .downloadExecutable(downloadUrl, executable, filenameShasum) + .then(process.exit) + .catch((err) => { + console.error(err); + process.exit(1); + }); +} diff --git a/ts-binary-wrapper/src/common.ts b/ts-binary-wrapper/src/common.ts new file mode 100644 index 0000000000..18da00141d --- /dev/null +++ b/ts-binary-wrapper/src/common.ts @@ -0,0 +1,254 @@ +import * as path from 'path'; +import * as os from 'os'; +import * as fs from 'fs'; +import { spawnSync } from 'child_process'; +import * as https from 'https'; +import { randomInt, createHash } from 'crypto'; + +export const versionFile = path.join(__dirname, 'generated', 'version'); +export const shasumFile = path.join(__dirname, 'generated', 'sha256sums.txt'); + +export class WrapperConfiguration { + private version: string; + private binaryName: string; + private expectedSha256sum: string; + + public constructor( + version: string, + binaryName: string, + expectedSha256sum: string, + ) { + this.version = version; + this.binaryName = binaryName; + this.expectedSha256sum = expectedSha256sum; + } + + public getVersion(): string { + return this.version; + } + + public getBinaryName(): string { + return this.binaryName; + } + + public getDownloadLocation(): string { + const baseUrl = 'https://static.snyk.io/cli/v'; + return baseUrl + this.version + '/' + this.binaryName; + } + + public getLocalLocation(): string { + const currentFolder = __dirname; + return path.join(currentFolder, this.binaryName); + } + + public getShasumFile(): string { + return this.expectedSha256sum; + } +} + +export function determineBinaryName( + platform: NodeJS.Platform, + arch: string, +): string { + const basename = 'snyk-'; + let osname: string; + let archname = ''; + let suffix = ''; + + switch (platform) { + case 'win32': + osname = 'win'; + suffix = '.exe'; + break; + case 'darwin': + osname = 'macos'; + break; + default: { + let isAlpine = false; + try { + const result = spawnSync('cat /etc/os-release', { shell: true }); + isAlpine = result.stdout + .toString() + .toLowerCase() + .includes('id=alpine'); + } catch { + isAlpine = false; + } + + if (isAlpine) { + osname = 'alpine'; + } else { + osname = 'linux'; + } + + break; + } + } + + switch (arch) { + case 'x64': + archname = ''; + break; + case 'arm64': + archname = '-arm64'; + break; + default: + throw 'Unsupported Architecture (' + arch + ')'; + } + + if (platform == 'linux') { + return basename + osname + archname + suffix; + } else { + return basename + osname + suffix; + } +} + +export function getCurrentVersion(filename: string): string { + try { + const version = fs.readFileSync(filename); + return version.toString().trim(); + } catch { + return ''; + } +} + +export function getCurrentSha256sum( + binaryName: string, + filename: string, +): string { + try { + const allsums = fs.readFileSync(filename).toString(); + const re = new RegExp('^([a-zA-Z0-9]+)[\\s\\*]+' + binaryName + '$', 'mig'); + const result = re.exec(allsums); + if (result) { + return result[1]; + } + } catch { + // + } + + return 'unknown-shasum-' + binaryName; +} + +export function getCurrentConfiguration(): WrapperConfiguration { + const binaryName = determineBinaryName(os.platform(), os.arch()); + const version = getCurrentVersion(versionFile); + const expectedSha256sum = getCurrentSha256sum(binaryName, shasumFile); + return new WrapperConfiguration(version, binaryName, expectedSha256sum); +} + +export function getCliArguments(inputArgv: string[]): string[] { + const cliArguments = inputArgv.slice(2); + return cliArguments; +} + +export function debugEnabled(cliArguments: string[]): boolean { + let debugIndex = cliArguments.indexOf('--debug'); + + if (debugIndex < 0) { + debugIndex = cliArguments.indexOf('-d'); + } + + return debugIndex >= 0; +} + +export function runWrapper(executable: string, cliArguments: string[]): number { + const debug = debugEnabled(cliArguments); + + if (debug) { + console.debug('Executing: ' + executable + ' ' + cliArguments.join(' ')); + } + + const res = spawnSync(executable, cliArguments, { + shell: false, + stdio: 'inherit', + }); + + if (debug) { + console.debug(res); + } + + let exitCode = 2; + if (res.status !== null) { + exitCode = res.status; + } else { + console.error( + 'Failed to spawn child process, ensure to run bootstrap first. (' + + executable + + ')', + ); + } + + return exitCode; +} + +export async function downloadExecutable( + downloadUrl: string, + filename: string, + filenameShasum: string, +): Promise { + return new Promise(function(resolve) { + const options = new URL(downloadUrl); + const temp = path.join(__dirname, randomInt(100000).toString()); + const fileStream = fs.createWriteStream(temp); + + const cleanupAfterError = (exitCode: number) => { + try { + fs.unlinkSync(temp); + } catch (e) { + console.debug('Failed to cleanup temporary file (' + temp + '): ' + e); + } + resolve(exitCode); + }; + + console.debug("Downloading from '" + downloadUrl + "' to '" + filename); + + const req = https.request(options, (res) => { + const shasum = createHash('sha256'); + res.pipe(fileStream); + res.pipe(shasum); + + fileStream.on('finish', () => { + fileStream.close(); + + // compare shasums + const actualShasum = shasum.digest('hex'); + console.debug( + 'Shasums:\n- actual: ' + + actualShasum + + '\n- expected: ' + + filenameShasum, + ); + if (filenameShasum && actualShasum != filenameShasum) { + console.error('Failed Shasum comparison!'); + cleanupAfterError(3); + } else { + // finally rename the file and change permissions + fs.renameSync(temp, filename); + fs.chmodSync(filename, 0o750); + console.debug('Downloaded successfull! '); + resolve(0); + } + }); + }); + + req.on('error', (e) => { + console.debug('Error during download!'); + console.error(e); + cleanupAfterError(1); + }); + + req.on('response', (incoming) => { + if ( + incoming.statusCode && + !(200 <= incoming.statusCode && incoming.statusCode < 300) + ) { + req.destroy(); + console.debug('Failed to download! ' + incoming.statusMessage); + cleanupAfterError(2); + } + }); + + req.end(); + }); +} diff --git a/ts-binary-wrapper/src/index.ts b/ts-binary-wrapper/src/index.ts new file mode 100644 index 0000000000..a89aac8059 --- /dev/null +++ b/ts-binary-wrapper/src/index.ts @@ -0,0 +1,10 @@ +#!/usr/bin/env node + +import { argv } from 'process'; +import * as common from './common'; + +const config = common.getCurrentConfiguration(); +const executable = config.getLocalLocation(); +const cliArguments = common.getCliArguments(argv); +const exitCode = common.runWrapper(executable, cliArguments); +process.exit(exitCode); diff --git a/ts-binary-wrapper/test/acceptance/basic.spec.ts b/ts-binary-wrapper/test/acceptance/basic.spec.ts new file mode 100644 index 0000000000..602b3fa47d --- /dev/null +++ b/ts-binary-wrapper/test/acceptance/basic.spec.ts @@ -0,0 +1,60 @@ +import { prepareEnvironment } from '../util/prepareEnvironment'; +import * as bootstrap from '../../src/bootstrap'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as child_process from 'child_process'; + +jest.setTimeout(60 * 1000); + +describe('Basic acceptance test', () => { + it('Bootstrap binary & execute a command', async () => { + const cliVersionForTesting = '1.1080.0'; + const envInfo = await prepareEnvironment(cliVersionForTesting); + const executable = bootstrap.executable.replace( + envInfo.inputfolder, + envInfo.outputfolder, + ); + + try { + fs.unlinkSync(executable); + } catch { + // + } + + expect(fs.existsSync(executable)).toBeFalsy(); + + const indexScript = path.join(envInfo.outputfolder, 'index.js'); + const bootstrapScript = path.join(envInfo.outputfolder, 'bootstrap.js'); + + // run system under test: index + const resultBootstrap = child_process.spawnSync( + 'node ' + bootstrapScript + ' exec', + { shell: true }, + ); + console.debug(resultBootstrap.stdout.toString()); + expect(resultBootstrap.status).toEqual(0); + expect(fs.existsSync(executable)).toBeTruthy(); + + // run system under test: index + const resultIndex = child_process.spawnSync( + 'node ' + indexScript + ' --version', + { shell: true }, + ); + + if (resultIndex.status != 0) { + console.debug(resultIndex); + } + + expect(resultIndex.status).toEqual(0); + expect(resultIndex.output.toString()).toContain(cliVersionForTesting); + + fs.unlinkSync(executable); + + try { + fs.rmSync(envInfo.outputfolder, { recursive: true }); + } catch { + // to support nodejs 12, which doesn't know rmSync() + fs.rmdirSync(envInfo.outputfolder, { recursive: true }); + } + }); +}); diff --git a/ts-binary-wrapper/test/unit/common.spec.ts b/ts-binary-wrapper/test/unit/common.spec.ts new file mode 100644 index 0000000000..2337b21767 --- /dev/null +++ b/ts-binary-wrapper/test/unit/common.spec.ts @@ -0,0 +1,266 @@ +import * as common from '../../src/common'; +import * as fs from 'fs'; +import * as path from 'path'; + +jest.setTimeout(60 * 1000); + +describe('Determine Binary Name', () => { + it('Determine Binary Name (darwin)', async () => { + const expected = 'snyk-macos'; + const actualx64 = common.determineBinaryName('darwin', 'x64'); + const actualarm64 = common.determineBinaryName('darwin', 'arm64'); + expect(actualx64).toEqual(expected); + expect(actualarm64).toEqual(expected); + }); + + it('Determine Binary Name (win)', async () => { + const expected = 'snyk-win.exe'; + const actualx64 = common.determineBinaryName('win32', 'x64'); + const actualarm64 = common.determineBinaryName('win32', 'arm64'); + expect(actualx64).toEqual(expected); + expect(actualarm64).toEqual(expected); + }); + + it('Determine Binary Name (linux)', async () => { + const expectedx64 = 'snyk-linux'; + const expectedarm64 = 'snyk-linux-arm64'; + const actualx64 = common.determineBinaryName('linux', 'x64'); + const actualarm64 = common.determineBinaryName('linux', 'arm64'); + expect(actualx64).toEqual(expectedx64); + expect(actualarm64).toEqual(expectedarm64); + }); + + it('Unsupported Architecture', async () => { + try { + common.determineBinaryName('linux', 'mipsel'); + } catch (e) { + expect(e).toBeDefined(); + } + }); +}); + +describe('Get Version', () => { + it('Version available', async () => { + const expected = '1.1080.0'; + const file = path.join(__dirname, 'test-version' + Math.random()); + fs.writeFileSync(file, '1.1080.0\n'); + + const actual = common.getCurrentVersion(file); + expect(actual).toEqual(expected); + + fs.unlinkSync(file); + }); + + it('Version file not available', async () => { + const expected = ''; + const file = path.join(__dirname, 'not-existing-file'); + + const actual = common.getCurrentVersion(file); + expect(actual).toEqual(expected); + }); +}); + +describe('Get Shasum', () => { + it('Shasum available (multiple)', async () => { + const expected = '0a238fe123'; + const file = path.join(__dirname, 'sha256sums.txt' + Math.random()); + fs.writeFileSync( + file, + '098fe123 *snyk-win\n12345 *snyk-macos\ncecece *snyk-linux-arm64\n0a238fe123 *snyk-linux', + ); + + const actual = common.getCurrentSha256sum('snyk-linux', file); + expect(actual).toEqual(expected); + + fs.unlinkSync(file); + }); + + it('Shasum available (single)', async () => { + const expected = '0a238fe123'; + const file = path.join(__dirname, 'sha256sums.txt' + Math.random()); + fs.writeFileSync(file, '0a238fe123 snyk-linux\n'); + + const actual = common.getCurrentSha256sum('snyk-linux', file); + expect(actual).toEqual(expected); + + fs.unlinkSync(file); + }); + + it('Shasum not available', async () => { + const expected = 'unknown-shasum-'; + const file = path.join(__dirname, 'sha256sums.txt' + Math.random()); + fs.writeFileSync( + file, + '098fe123 *snyk-win\n12345 *snyk-macos\n0a238fe123 *snyk-linux', + ); + + const actual = common.getCurrentSha256sum('snyk-linux-arm64', file); + expect(actual).toContain(expected); + + fs.unlinkSync(file); + }); +}); + +describe('Configuration', () => { + it('Download and local location', async () => { + const expectedDownloadLocation = + 'https://static.snyk.io/cli/v1.2.3/snyk-win.exe'; + const expectedLocalLocation = path.join( + __dirname, + '..', + '..', + 'src', + 'snyk-win.exe', + ); + const config = new common.WrapperConfiguration( + '1.2.3', + 'snyk-win.exe', + '1234abcdef', + ); + + const actualDownloadLocation = config.getDownloadLocation(); + expect(actualDownloadLocation).toEqual(expectedDownloadLocation); + + const actualLocalLocation = config.getLocalLocation(); + expect(actualLocalLocation).toEqual(expectedLocalLocation); + }); +}); + +describe('Testing binary wrapper', () => { + it('getCliArguments() filter important stuff', async () => { + const indexFile = path.join(__dirname, '..', '..', 'src', 'index.ts'); + const input = ['ignore', indexFile, 'important', 'stuff']; + const expected = ['important', 'stuff']; + const actual = common.getCliArguments(input); + expect(actual).toEqual(expected); + }); + + it('getCliArguments() filter important stuff (with directory only)', async () => { + const indexFile = path.join(__dirname, '..', '..', 'src'); + const input = ['ignore', indexFile, 'important', 'stuff']; + const expected = ['important', 'stuff']; + const actual = common.getCliArguments(input); + expect(actual).toEqual(expected); + }); + + it('runWrapper() succesfully', async () => { + const executable = 'node'; + const cliArguments = ['--version']; + const expected = 0; + const actual = common.runWrapper(executable, cliArguments); + expect(actual).toEqual(expected); + }); + + it('runWrapper() fail', async () => { + const executable = 'node-unknown'; + const cliArguments = ['--version']; + const expected = 2; + const actual = common.runWrapper(executable, cliArguments); + expect(actual).toEqual(expected); + }); + + it('debugEnabled() false', async () => { + const cliArguments = ['--version', '--something', 'else']; + const expected = false; + const actual = common.debugEnabled(cliArguments); + expect(actual).toEqual(expected); + }); + + it('debugEnabled() true (--debug)', async () => { + const cliArguments = ['--version', '--something', '--debug', 'else']; + const expected = true; + const actual = common.debugEnabled(cliArguments); + expect(actual).toEqual(expected); + }); + + it('debugEnabled() true (-d)', async () => { + const cliArguments = ['--version', '--something', '-d', 'else']; + const expected = true; + const actual = common.debugEnabled(cliArguments); + expect(actual).toEqual(expected); + }); +}); + +describe('Testing binary bootstrapper', () => { + it('downloadExecutable() succesfull', async () => { + const binaryName = 'snyk-macos'; + const shafileExtension = '.sha256'; + const config = new common.WrapperConfiguration('1.1080.0', binaryName, ''); + const shasumFile = + config.getLocalLocation() + Math.random() + shafileExtension; + + // download the shasum first, here we don't expect a shasum comparison + const shasumDownload = await common.downloadExecutable( + config.getDownloadLocation() + shafileExtension, + shasumFile, + '', + ); + expect(shasumDownload).toEqual(0); + expect(fs.existsSync(shasumFile)).toBeTruthy(); + const expectedShasum = common.getCurrentSha256sum(binaryName, shasumFile); + + // download binary next and use previously downloaded shasum to check validity + const binaryDownload = await common.downloadExecutable( + config.getDownloadLocation(), + config.getLocalLocation(), + expectedShasum, + ); + expect(binaryDownload).toEqual(0); + expect(fs.existsSync(config.getLocalLocation())).toBeTruthy(); + + try { + // check if the binary is executable + fs.accessSync(config.getLocalLocation(), fs.constants.X_OK); + } catch { + // execution of binary not possible + expect(false).toBeTruthy(); + } + + fs.unlinkSync(shasumFile); + fs.unlinkSync(config.getLocalLocation()); + }); + + it('downloadExecutable() fails due to incorrect shasum', async () => { + const binaryName = 'snyk-macos'; + const shafileExtension = '.sha256'; + const config = new common.WrapperConfiguration('1.1080.0', binaryName, ''); + const shasumFile = + config.getLocalLocation() + Math.random() + shafileExtension; + + // download just any file and state a shasum expectation that never can be fullfilled + const shasumDownload = await common.downloadExecutable( + config.getDownloadLocation() + shafileExtension, + shasumFile, + 'incorrect-shasum', + ); + expect(shasumDownload).toEqual(3); + expect(fs.existsSync(shasumFile)).toBeFalsy(); + }); + + it("downloadExecutable() try to download a file that doesn't exist", async () => { + const binaryName = 'snyk-macos'; + const shafileExtension = '.shoe256'; + const config = new common.WrapperConfiguration('1.1080.0', binaryName, ''); + const shasumFile = + config.getLocalLocation() + Math.random() + shafileExtension; + + // try to download a file that doesn't exis + const shasumDownload = await common.downloadExecutable( + config.getDownloadLocation() + shafileExtension, + shasumFile, + 'incorrect-shasum', + ); + expect(shasumDownload).toEqual(2); + expect(fs.existsSync(shasumFile)).toBeFalsy(); + }); + + it('downloadExecutable() fails due to an error in the https connection', async () => { + // download the just any file and state a shasum expectation that never can be fullfilled + const shasumDownload = await common.downloadExecutable( + 'https://notaurl', + '', + '', + ); + expect(shasumDownload).toEqual(1); + }); +}); diff --git a/ts-binary-wrapper/test/util/prepareEnvironment.ts b/ts-binary-wrapper/test/util/prepareEnvironment.ts new file mode 100644 index 0000000000..040d0ddeee --- /dev/null +++ b/ts-binary-wrapper/test/util/prepareEnvironment.ts @@ -0,0 +1,54 @@ +import * as common from '../../src/common'; +import * as fs from 'fs'; +import * as child_process from 'child_process'; +import * as path from 'path'; + +interface TestEnvironmentInfo { + inputfolder: string; + outputfolder: string; +} + +export async function prepareEnvironment( + version: string, +): Promise { + const inputfolder = path.join(__dirname, '..', '..', 'src'); + const outputfolder = path.join(__dirname, 'something'); + const versionFile = common.versionFile.replace(inputfolder, outputfolder); + const shasumFile = common.shasumFile.replace(inputfolder, outputfolder); + + if (fs.existsSync(outputfolder)) { + fs.rmSync(outputfolder, { recursive: true }); + } + + fs.mkdirSync(path.join(outputfolder, 'generated'), { recursive: true }); + + const tsc = child_process.spawnSync( + 'tsc --outDir ' + + outputfolder + + ' --tsBuildInfoFile ' + + path.join(outputfolder, 'tsconfig.tsbuildinfo'), + { cwd: inputfolder, shell: true }, + ); + + if (tsc.status) { + console.debug(tsc); + console.debug(tsc.stdout.toString()); + console.debug(tsc.stderr.toString()); + } + + fs.writeFileSync(versionFile, version); + + await common.downloadExecutable( + 'https://static.snyk.io/cli/v' + version + '/sha256sums.txt.asc', + shasumFile, + '', + ); + + return { inputfolder, outputfolder }; +} + +if (process.argv.includes('exec')) { + (async function() { + await prepareEnvironment('1.1080.0'); + }); +} diff --git a/ts-binary-wrapper/tsconfig.json b/ts-binary-wrapper/tsconfig.json new file mode 100644 index 0000000000..12e57d05b2 --- /dev/null +++ b/ts-binary-wrapper/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../tsconfig.settings.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["./src/**/*"], + "types": ["node", "jest"] +}