From 62abaea1d65735434a9f12cb36720260945f17a1 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Tue, 17 Dec 2024 22:44:07 -0800 Subject: [PATCH 01/44] Add node test for checking exported symbols Signed-off-by: Prateek Kumar --- node/DEVELOPER.md | 65 +++++--- node/README.md | 8 +- node/npm/glide/index.ts | 108 +++++++------ node/npm/glide/jest.config.js | 33 ++++ node/npm/glide/tests/ExportedSymbols.test.ts | 156 +++++++++++++++++++ node/npm/glide/tests/tsconfig.json | 7 + node/rust-client/src/lib.rs | 9 ++ node/src/BaseClient.ts | 4 + node/src/Commands.ts | 3 +- node/src/Transaction.ts | 2 +- node/src/server-modules/GlideFtOptions.ts | 2 +- 11 files changed, 317 insertions(+), 80 deletions(-) create mode 100644 node/npm/glide/jest.config.js create mode 100644 node/npm/glide/tests/ExportedSymbols.test.ts create mode 100644 node/npm/glide/tests/tsconfig.json diff --git a/node/DEVELOPER.md b/node/DEVELOPER.md index 8878fdd91d..69be8ea15f 100644 --- a/node/DEVELOPER.md +++ b/node/DEVELOPER.md @@ -16,14 +16,14 @@ Software Dependencies If your Nodejs version is below the supported version specified in the client's [documentation](https://github.com/valkey-io/valkey-glide/blob/main/node/README.md#nodejs-supported-version), you can upgrade it using [NVM](https://github.com/nvm-sh/nvm?tab=readme-ov-file#install--update-script). -- npm -- git -- GCC -- pkg-config -- protoc (protobuf compiler) -- openssl -- openssl-dev -- rustup +- npm +- git +- GCC +- pkg-config +- protoc (protobuf compiler) +- openssl +- openssl-dev +- rustup **Dependencies installation for Ubuntu** @@ -107,16 +107,36 @@ Before starting this step, make sure you've installed all software requirments. 5. Integrating the built GLIDE package into your project: Add the package to your project using the folder path with the command `npm install /node`. -- For a fast build, execute `npm run build`. This will perform a full, unoptimized build, which is suitable for developing tests. Keep in mind that performance is significantly affected in an unoptimized build, so it's required to build with the `build:release` or `build:benchmark` option when measuring performance. -- If your modifications are limited to the TypeScript code, run `npm run build-external` to build the external package without rebuilding the internal package. -- If your modifications are limited to the Rust code, execute `npm run build-internal` to build the internal package and generate TypeScript code. -- To generate Node's protobuf files, execute `npm run build-protobuf`. Keep in mind that protobuf files are generated as part of full builds (e.g., `build`, `build:release`, etc.). +6. Testing the GLIDE npm package: + The `node/npm/glide` folder contains a package wrapper that re-exports all the native bindings. To build and test this package, follow these steps: + ```bash + + Build node package: + co node + export nativeStr=darwin-x64; export scope=@valkey/; + envsubst < package.json.tmpl > "package.json" + npm run build + npm run build:release + In npm/glide package run the following commands: + cd ../npm/glide + export pkg_name=valkey-glide; export package_version=99.99.0; export scope=@valkey/; + envsubst < package.json.tmpl > "package.json" + npm run build + npm run build:test + npm run test + npm run test -- --testNamePattern="Exported symbols test" + ``` + +- For a fast build, execute `npm run build`. This will perform a full, unoptimized build, which is suitable for developing tests. Keep in mind that performance is significantly affected in an unoptimized build, so it's required to build with the `build:release` or `build:benchmark` option when measuring performance. +- If your modifications are limited to the TypeScript code, run `npm run build-external` to build the external package without rebuilding the internal package. +- If your modifications are limited to the Rust code, execute `npm run build-internal` to build the internal package and generate TypeScript code. +- To generate Node's protobuf files, execute `npm run build-protobuf`. Keep in mind that protobuf files are generated as part of full builds (e.g., `build`, `build:release`, etc.). > Note: Once building completed, you'll find the compiled JavaScript code in the `node/build-ts` folder. ### Troubleshooting -- If the build fails after running `npx tsc` because `glide-rs` isn't found, check if your npm version is in the range 9.0.0-9.4.1, and if so, upgrade it. 9.4.2 contains a fix to a change introduced in 9.0.0 that is required in order to build the library. +- If the build fails after running `npx tsc` because `glide-rs` isn't found, check if your npm version is in the range 9.0.0-9.4.1, and if so, upgrade it. 9.4.2 contains a fix to a change introduced in 9.0.0 that is required in order to build the library. ### Test @@ -156,6 +176,7 @@ In order to run these tests, use: ```bash npm run test-modules -- --cluster-endpoints=
: ``` + Note: these tests don't run with standalone server as of now. ### REPL (interactive shell) @@ -201,13 +222,13 @@ Development on the Node wrapper may involve changes in either the TypeScript or **TypeScript:** -- ESLint -- Prettier +- ESLint +- Prettier **Rust:** -- clippy -- fmt +- clippy +- fmt #### Running the linters @@ -230,8 +251,8 @@ Development on the Node wrapper may involve changes in either the TypeScript or ### Recommended extensions for VS Code -- [Prettier - Code formatter](https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode) - JavaScript / TypeScript formatter. -- [ESLint](https://marketplace.visualstudio.com/items?itemName=dbaeumer.vscode-eslint) - linter. -- [Jest Runner](https://marketplace.visualstudio.com/items?itemName=firsttris.vscode-jest-runner) - in-editor test runner. -- [Jest Test Explorer](https://marketplace.visualstudio.com/items?itemName=kavod-io.vscode-jest-test-adapter) - adapter to the VSCode testing UI. -- [rust-analyzer](https://marketplace.visualstudio.com/items?itemName=rust-lang.rust-analyzer) - Rust language support for VSCode. +- [Prettier - Code formatter](https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode) - JavaScript / TypeScript formatter. +- [ESLint](https://marketplace.visualstudio.com/items?itemName=dbaeumer.vscode-eslint) - linter. +- [Jest Runner](https://marketplace.visualstudio.com/items?itemName=firsttris.vscode-jest-runner) - in-editor test runner. +- [Jest Test Explorer](https://marketplace.visualstudio.com/items?itemName=kavod-io.vscode-jest-test-adapter) - adapter to the VSCode testing UI. +- [rust-analyzer](https://marketplace.visualstudio.com/items?itemName=rust-lang.rust-analyzer) - Rust language support for VSCode. diff --git a/node/README.md b/node/README.md index eeaf4e4a60..299499d202 100644 --- a/node/README.md +++ b/node/README.md @@ -14,16 +14,16 @@ The release of Valkey GLIDE was tested on the following platforms: Linux: -- Ubuntu 22.04.1 (x86_64 and aarch64) -- Amazon Linux 2023 (AL2023) (x86_64) +- Ubuntu 22.04.1 (x86_64 and aarch64) +- Amazon Linux 2023 (AL2023) (x86_64) macOS: -- macOS 14.7 (Apple silicon/aarch_64) +- macOS 14.7 (Apple silicon/aarch_64) Alpine: -- node:alpine (default on aarch64 and x86_64) +- node:alpine (default on aarch64 and x86_64) ## NodeJS supported version diff --git a/node/npm/glide/index.ts b/node/npm/glide/index.ts index c4dab9795b..277c6b0412 100644 --- a/node/npm/glide/index.ts +++ b/node/npm/glide/index.ts @@ -11,64 +11,70 @@ let globalObject = global as unknown; /* eslint-disable @typescript-eslint/no-require-imports */ function loadNativeBinding() { - let nativeBinding = null; + const scope = process.env.scope || "@scope"; - switch (platform) { - case "linux": - switch (arch) { - case "x64": - switch (familySync()) { - case GLIBC: - nativeBinding = require("@scope/valkey-glide-linux-x64"); - break; - case MUSL: - nativeBinding = require("@scope/valkey-glide-linux-musl-x64"); - break; - default: - nativeBinding = require("@scope/valkey-glide-linux-x64"); - break; - } + let nativeBinding = []; + let nativeStr = process.env.native_binding; - break; - case "arm64": - switch (familySync()) { - case GLIBC: - nativeBinding = require("@scope/valkey-glide-linux-arm64"); - break; - case MUSL: - nativeBinding = require("@scope/valkey-glide-linux-musl-arm64"); - break; - default: - nativeBinding = require("@scope/valkey-glide-linux-arm64"); - break; - } + if (nativeStr == undefined) { + switch (platform) { + case "linux": + switch (arch) { + case "x64": + switch (familySync()) { + case MUSL: + nativeStr = "linux-musl-x64"; + break; + case GLIBC: + default: + nativeStr = "linux-x64"; + break; + } - break; - default: - throw new Error( - `Unsupported OS: ${platform}, architecture: ${arch}`, - ); - } + break; + case "arm64": + switch (familySync()) { + case MUSL: + nativeStr = "linux-musl-arm64"; + break; + case GLIBC: + default: + nativeStr = "linux-arm64"; + break; + } - break; - case "darwin": - switch (arch) { - case "arm64": - nativeBinding = require("@scope/valkey-glide-darwin-arm64"); - break; - default: - throw new Error( - `Unsupported OS: ${platform}, architecture: ${arch}`, - ); - } + break; + default: + throw new Error( + `Unsupported OS: ${platform}, architecture: ${arch}`, + ); + } - break; - default: - throw new Error( - `Unsupported OS: ${platform}, architecture: ${arch}`, - ); + break; + case "darwin": + switch (arch) { + case "x64": + nativeStr = "darwin-x64"; + break; + case "arm64": + nativeStr = "darwin-arm64"; + break; + default: + throw new Error( + `Unsupported OS: ${platform}, architecture: ${arch}`, + ); + } + + break; + default: + throw new Error( + `Unsupported OS: ${platform}, architecture: ${arch}`, + ); + } } + nativeBinding = require(`${scope}valkey-glide-${nativeStr}`); + if (!nativeBinding) { throw new Error(`Failed to load native binding`); } diff --git a/node/npm/glide/jest.config.js b/node/npm/glide/jest.config.js new file mode 100644 index 0000000000..c68ff7fa63 --- /dev/null +++ b/node/npm/glide/jest.config.js @@ -0,0 +1,33 @@ + +/* eslint no-undef: off */ +module.exports = { + preset: "ts-jest", + transform: { + "^.+\\.(t|j)s$": ["ts-jest", { isolatedModules: true }], + }, + testEnvironment: "node", + testRegex: "/tests/.*\\.(test|spec)?\\.(ts|tsx)$", + moduleFileExtensions: [ + "ts", + "tsx", + "js", + "jsx", + "json", + "node", + "cjs", + "mjs", + ], + testTimeout: 600000, + reporters: [ + "default", + [ + "./node_modules/jest-html-reporter", + { + includeFailureMsg: true, + includeSuiteFailure: true, + executionTimeWarningThreshold: 60, + sort: "status", + }, + ], + ] +}; diff --git a/node/npm/glide/tests/ExportedSymbols.test.ts b/node/npm/glide/tests/ExportedSymbols.test.ts new file mode 100644 index 0000000000..058f01998c --- /dev/null +++ b/node/npm/glide/tests/ExportedSymbols.test.ts @@ -0,0 +1,156 @@ +import { it } from "@jest/globals"; +import * as f from "fs/promises"; +import { describe } from "node:test"; +import * as ts from "typescript"; +import * as glideApi from "../"; + +describe("Exported Symbols test", () => { + it("check excluded symbols are not exported", async () => { + // Check exported symbols for valkey glide package + const exportedSymbolsList = Object.keys(glideApi).sort(); // exportedList + + const implBuildFolder = "./build-ts/"; + // const rustClientBuildFolder = './node_modules/@valkey/valkey-glide-impl/rust-client/'; + // const filesWithNodeCode = [...await getFiles(implBuildFolder), ...await getFiles(rustClientBuildFolder)]; + const filesWithNodeCode = await getFiles(implBuildFolder); + console.log(filesWithNodeCode); + + const internallyExported: any = []; + + for (const file of filesWithNodeCode) { + const sourceCode = await f.readFile(file, "utf8"); + const sourceFile = await ts.createSourceFile( + file, + sourceCode, + ts.ScriptTarget.Latest, + true, + ); + internallyExported.push(...visitRoot(sourceFile)); + } + + internallyExported.sort(); + + let missingSymbols = internallyExported.filter( + (e: string) => !exportedSymbolsList.includes(e), + ); + let doesNotExistExports = exportedSymbolsList.filter( + (e: string | any) => !internallyExported.includes(e), + ); + console.log(missingSymbols); + console.log(doesNotExistExports); + expect(exportedSymbolsList).toEqual(internallyExported); + }); +}); + +async function getFiles(folderName: string): Promise { + const files = await f.readdir(folderName, { withFileTypes: true }); + + const skipFolders = [ + "commonjs-test", + "glide-logs", + "hybrid-node-tests", + "node_modules", + "npm", + ".cargo", + "target", + "tests", + ]; + + const filesWithNodeCode = []; + + for (const file of files) { + if (file.isDirectory()) { + if (skipFolders.includes(file.name)) { + continue; + } + + filesWithNodeCode.push( + ...(await getFiles(folderName + file.name + "/")), + ); + } else { + if (!file.name.endsWith(".d.ts")) { + continue; + } + filesWithNodeCode.push(folderName + file.name); + } + } + + return filesWithNodeCode; +} + +function visitRoot(root: ts.Node) { + // (Root Level)->(Level 1) + const children: ts.Node[] = root.getChildren(); + + const resultList: string[] = []; + // (Root Level) -> (Level 1) -> Level 2. This is the level in the AST where all the exported symbols in a file are present. + for (const node of children) { + const nodeList: string[] = node + .getChildren() + .map((c) => visit(c)) + .filter((c) => c !== undefined); + + if (nodeList.length > 0) { + resultList.push(...nodeList); + } + } + + return resultList; +} + +function visit(node: ts.Node) { + let name: string | undefined = ""; + + // List of exported symbols we want to ignore. + switch (node.kind) { + case ts.SyntaxKind.FirstStatement: + case ts.SyntaxKind.ExportDeclaration: + case ts.SyntaxKind.ExportAssignment: + case ts.SyntaxKind.ImportDeclaration: + return; + } + + // list exported symbols we want to check for, like, InterfaceDeclaration, FunctionDeclaration, etc. + if (ts.isFunctionDeclaration(node)) { + name = node.name?.text; + } else if (ts.isVariableStatement(node)) { + name = ""; + } else if (ts.isInterfaceDeclaration(node)) { + name = node.name?.text; + } else if (ts.isClassDeclaration(node)) { + name = node.name?.text; + } else if (ts.isTypeAliasDeclaration(node)) { + name = node.name?.text; + } else if (ts.isEnumDeclaration(node)) { + name = node.name?.text; + } else if (ts.isModuleDeclaration(node)) { + name = node.name?.text; + } + + const children = node.getChildren(); + const isInternal = children + .find((c) => ts.SyntaxKind[c.kind] == "JSDocComment") + ?.getText() + .includes("@internal"); + const isExported = children + .find((c) => ts.SyntaxKind[c.kind] == "SyntaxList") + ?.getChildren() + .find((c) => ts.SyntaxKind[c.kind] == "ExportKeyword"); + + if (isExported && !isInternal) { + // Not internal symbol exported for external use. + return name; + } + + if (isExported && isInternal) { + // marked correctly... no-op. Exported for internal use in the code. + } + + if (!isExported && isInternal) { + // no-op + } + + if (!isExported && !isInternal) { + // no-op + } +} diff --git a/node/npm/glide/tests/tsconfig.json b/node/npm/glide/tests/tsconfig.json new file mode 100644 index 0000000000..2a8aaebf40 --- /dev/null +++ b/node/npm/glide/tests/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "rootDir": "../" + }, + "include": ["./*.test.ts"] +} diff --git a/node/rust-client/src/lib.rs b/node/rust-client/src/lib.rs index 963f966f24..165724b5d6 100644 --- a/node/rust-client/src/lib.rs +++ b/node/rust-client/src/lib.rs @@ -306,6 +306,7 @@ fn split_pointer(pointer: *mut T) -> [u32; 2] { } #[napi(ts_return_type = "[number, number]")] +/// @internal @test /// This function is for tests that require a value allocated on the heap. /// Should NOT be used in production. #[cfg(feature = "testing_utilities")] @@ -316,6 +317,9 @@ pub fn create_leaked_string(message: String) -> [u32; 2] { } #[napi(ts_return_type = "[number, number]")] +/// @internal @test +/// This function is for tests that require a value allocated on the heap. +/// Should NOT be used in production. pub fn create_leaked_string_vec(message: Vec) -> [u32; 2] { // Convert the string vec -> Bytes vector let bytes_vec: Vec = message.iter().map(|v| Bytes::from(v.to_vec())).collect(); @@ -324,6 +328,7 @@ pub fn create_leaked_string_vec(message: Vec) -> [u32; 2] { } #[napi(ts_return_type = "[number, number]")] +/// @internal @test /// This function is for tests that require a value allocated on the heap. /// Should NOT be used in production. #[cfg(feature = "testing_utilities")] @@ -337,6 +342,7 @@ pub fn create_leaked_map(map: HashMap) -> [u32; 2] { } #[napi(ts_return_type = "[number, number]")] +/// @internal @test /// This function is for tests that require a value allocated on the heap. /// Should NOT be used in production. #[cfg(feature = "testing_utilities")] @@ -348,6 +354,7 @@ pub fn create_leaked_array(array: Vec) -> [u32; 2] { } #[napi(ts_return_type = "[number, number]")] +/// @internal @test /// This function is for tests that require a value allocated on the heap. /// Should NOT be used in production. #[cfg(feature = "testing_utilities")] @@ -363,6 +370,7 @@ pub fn create_leaked_attribute(message: String, attribute: HashMap [u32; 2] { } #[napi(ts_return_type = "[number, number]")] +/// @internal @test /// This function is for tests that require a value allocated on the heap. /// Should NOT be used in production. #[cfg(feature = "testing_utilities")] diff --git a/node/src/BaseClient.ts b/node/src/BaseClient.ts index ded6ea58f3..e75c89fb3a 100644 --- a/node/src/BaseClient.ts +++ b/node/src/BaseClient.ts @@ -776,6 +776,10 @@ export interface PubSubMsg { */ export type WritePromiseOptions = RouteOption & DecoderOption; +/** + * @internal + * Base client interface for GLIDE + */ export class BaseClient { private socket: net.Socket; protected readonly promiseCallbackFunctions: [ diff --git a/node/src/Commands.ts b/node/src/Commands.ts index e7e10d7e5f..8a126a1abf 100644 --- a/node/src/Commands.ts +++ b/node/src/Commands.ts @@ -54,7 +54,7 @@ function toBuffersArray(args: GlideString[]) { } /** - * @internal + * @internal @test */ export function parseInfoResponse(response: string): Record { const lines = response.split("\n"); @@ -418,6 +418,7 @@ export function createHGet( } /** + * @internal * This function converts an input from {@link HashDataType} or `Record` types to `HashDataType`. * * @param fieldsAndValues - field names and their values. diff --git a/node/src/Transaction.ts b/node/src/Transaction.ts index bdccbe151f..19dd232f13 100644 --- a/node/src/Transaction.ts +++ b/node/src/Transaction.ts @@ -287,7 +287,7 @@ import { command_request } from "./ProtobufMessage"; * console.log(result); // Output: ['OK', 'value'] * ``` */ -export class BaseTransaction> { +class BaseTransaction> { /** * @internal */ diff --git a/node/src/server-modules/GlideFtOptions.ts b/node/src/server-modules/GlideFtOptions.ts index 6d9e8f4528..e5c16e0e7d 100644 --- a/node/src/server-modules/GlideFtOptions.ts +++ b/node/src/server-modules/GlideFtOptions.ts @@ -58,7 +58,7 @@ export type VectorField = BaseField & { /** * Base class for defining vector field attributes to be used after the vector algorithm name. */ -export interface VectorFieldAttributes { +interface VectorFieldAttributes { /** Number of dimensions in the vector. Equivalent to `DIM` in the module API. */ dimensions: number; /** From 9bfb65dbc7e8227f6de057e15f359683b1328eb8 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Tue, 17 Dec 2024 22:49:20 -0800 Subject: [PATCH 02/44] Add package.json.tmpl file Signed-off-by: Prateek Kumar --- node/npm/glide/package.json.tmpl | 75 ++++++++++++++++++++++++ node/package.json.tmpl | 97 ++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 node/npm/glide/package.json.tmpl create mode 100644 node/package.json.tmpl diff --git a/node/npm/glide/package.json.tmpl b/node/npm/glide/package.json.tmpl new file mode 100644 index 0000000000..bc5ae26eb7 --- /dev/null +++ b/node/npm/glide/package.json.tmpl @@ -0,0 +1,75 @@ +{ + "name": "${scope}${pkg_name}", + "types": "build-ts/index.d.ts", + "version": "${package_version}", + "description": "General Language Independent Driver for the Enterprise (GLIDE) for Valkey", + "main": "build-ts/index.js", + "module": "build-ts/index.js", + "type": "commonjs", + "scripts": { + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "clean": "rm -rf build-ts/", + "copy-declaration-files": "cp ../../build-ts/*.d.ts build-ts/ && cp ../../build-ts/src/*.d.ts build-ts/src/ && cp ../../build-ts/src/server-modules/*.d.ts build-ts/src/server-modules/", + "build": "tsc && mkdir -p build-ts/src && mkdir -p build-ts/src/server-modules && npm run copy-declaration-files", + "build:test": "npm i && npm run build", + "test": "jest --verbose" + }, + "files": [ + "/build-ts" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/valkey-io/valkey-glide.git" + }, + "keywords": [ + "valkey", + "valkeyClient", + "client", + "valkey-glide" + ], + "author": "Valkey GLIDE Maintainers", + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/valkey-io/valkey-glide/issues" + }, + "homepage": "https://github.com/valkey-io/valkey-glide#readme", + "devDependencies": { + "@jest/globals": "^29.7.0", + "@types/jest": "^29.5.14", + "ts-jest": "^29.2.5", + "jest": "^29.7.0", + "jest-html-reporter": "^3.10.2", + "@types/node": "^18.11.18", + "@typescript-eslint/eslint-plugin": "^5.48.0", + "@typescript-eslint/parser": "^5.48.0", + "eslint": "^8.31.0", + "typescript": "^4.9.4", + "${scope}valkey-glide-${nativeStr}": "../.." + }, + "optionalDependencies": { + "${scope}valkey-glide-darwin-arm64": "${package_version}", + "${scope}valkey-glide-darwin-x64": "${package_version}", + "${scope}valkey-glide-linux-arm64": "${package_version}", + "${scope}valkey-glide-linux-x64": "${package_version}", + "${scope}valkey-glide-linux-musl-arm64": "${package_version}", + "${scope}valkey-glide-linux-musl-x64": "${package_version}" + }, + "eslintConfig": { + "extends": [ + "eslint:recommended", + "plugin:@typescript-eslint/recommended" + ], + "parser": "@typescript-eslint/parser", + "plugins": [ + "@typescript-eslint" + ], + "ignorePatterns": [ + "build-ts/*" + ], + "root": true + }, + "dependencies": { + "detect-libc": "^2.0.3" + } +} diff --git a/node/package.json.tmpl b/node/package.json.tmpl new file mode 100644 index 0000000000..6a8b50b96c --- /dev/null +++ b/node/package.json.tmpl @@ -0,0 +1,97 @@ +{ + "name": "@valkey/valkey-glide-impl", + "description": "General Language Independent Driver for the Enterprise (GLIDE) for Valkey", + "main": "build-ts/index.js", + "module": "build-ts/index.js", + "types": "./build-ts/index.d.ts", + "type": "commonjs", + "repository": { + "type": "git", + "url": "git+https://github.com/valkey-io/valkey-glide.git" + }, + "homepage": "https://github.com/valkey-io/valkey-glide#readme", + "dependencies": { + "glide-rs": "file:rust-client", + "long": "^5.2.3", + "npmignore": "^0.3.1", + "protobufjs": "^7.4.0" + }, + "bundleDependencies": [ + "glide-rs" + ], + "scripts": { + "build": "npm run prereq && npm run build-internal && npm run build-protobuf && npm run build-external", + "build:release": "npm run build-internal:release && npm run build-protobuf && npm run build-external:release", + "build:benchmark": "npm run build-internal:benchmark && npm run build-protobuf && npm run build-external", + "build-internal": "cd rust-client && npm run build", + "build-internal:release": "cd rust-client && npm run build:release", + "build-internal:benchmark": "cd rust-client && npm run build:benchmark", + "build-external": "rm -rf build-ts && tsc", + "build-external:release": "rm -rf build-ts && tsc --stripInternal", + "build-protobuf": "npm run compile-protobuf-files && npm run fix-protobuf-file", + "compile-protobuf-files": "cd src && pbjs -t static-module -o ProtobufMessage.js ../../glide-core/src/protobuf/*.proto && pbts -o ProtobufMessage.d.ts ProtobufMessage.js", + "clean": "rm -rf build-ts rust-client/target docs glide-logs rust-client/glide-rs.*.node rust-client/index.* src/ProtobufMessage.*", + "fix-protobuf-file": "replace 'this\\.encode\\(message, writer\\)\\.ldelim' 'this.encode(message, writer && writer.len ? writer.fork() : writer).ldelim' src/ProtobufMessage.js", + "test": "npm run build-test-utils && jest --verbose --testPathIgnorePatterns='ServerModules'", + "test-dbg": "npm run build-test-utils && jest --runInBand", + "test-minimum": "npm run build-test-utils && jest --verbose --testNamePattern='^(.(?!(GlideJson|GlideFt|pubsub|kill)))*$'", + "test-modules": "npm run build-test-utils && jest --verbose --testNamePattern='(GlideJson|GlideFt)'", + "build-test-utils": "cd ../utils && npm i && npm run build", + "lint:fix": "npm run install-linting && npx eslint -c ../eslint.config.mjs --fix && npm run prettier:format", + "lint": "npm run install-linting && npx eslint -c ../eslint.config.mjs && npm run prettier:check:ci", + "install-linting": "cd ../ & npm install", + "prepack": "npmignore --auto", + "prereq": "npm install", + "prettier:check:ci": "npx prettier --check . --ignore-unknown '!**/*.{js,d.ts}'", + "prettier:format": "npx prettier --write . --ignore-unknown '!**/*.{js,d.ts}'" + }, + "devDependencies": { + "@jest/globals": "^29.7.0", + "@types/jest": "^29.5.14", + "@types/minimist": "^1.2.5", + "@types/redis-server": "^1.2.2", + "@types/semver": "^7.5.8", + "@types/uuid": "^10.0.0", + "find-free-port": "^2.0.0", + "jest": "^29.7.0", + "jest-html-reporter": "^3.10.2", + "protobufjs-cli": "^1.1.3", + "redis-server": "^1.2.2", + "replace": "^1.2.2", + "semver": "^7.6.3", + "ts-jest": "^29.2.5", + "ts-node": "^10.9.2", + "typescript": "^5.6.3", + "uuid": "^11.0.3" + }, + "author": "Valkey GLIDE Maintainers", + "license": "Apache-2.0", + "publishConfig": { + "${registry_scope}registry": "https://registry.npmjs.org/", + "ignore": [ + "src/**", + "tests/", + "rust-client/**", + "!build-ts/**", + ".prettierignore", + "jest.config.js", + "hybrid-node-tests/**", + "docs/", + "DEVELOPER.md", + ".ort.yml", + "tsconfig.json", + "THIRD_PARTY_LICENSES_NODE" + ] + }, + "//": [ + "The fields below have been commented out and are only necessary for publishing the package." + ], + "///cpu": [ + "${node_arch}" + ], + "///os": [ + "${node_os}" + ], + "///name": "${scope}valkey-glide-${nativeStr}", + "///version": "${package_version}" +} From be7925521942f8b2b27a4582dbfd46a3c0b19f47 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Tue, 17 Dec 2024 22:57:49 -0800 Subject: [PATCH 03/44] Fix formatting Signed-off-by: Prateek Kumar --- node/npm/glide/tests/ExportedSymbols.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/node/npm/glide/tests/ExportedSymbols.test.ts b/node/npm/glide/tests/ExportedSymbols.test.ts index 058f01998c..692b748f56 100644 --- a/node/npm/glide/tests/ExportedSymbols.test.ts +++ b/node/npm/glide/tests/ExportedSymbols.test.ts @@ -15,7 +15,7 @@ describe("Exported Symbols test", () => { const filesWithNodeCode = await getFiles(implBuildFolder); console.log(filesWithNodeCode); - const internallyExported: any = []; + const internallyExported: any[] = []; for (const file of filesWithNodeCode) { const sourceCode = await f.readFile(file, "utf8"); @@ -33,8 +33,8 @@ describe("Exported Symbols test", () => { let missingSymbols = internallyExported.filter( (e: string) => !exportedSymbolsList.includes(e), ); - let doesNotExistExports = exportedSymbolsList.filter( - (e: string | any) => !internallyExported.includes(e), + const doesNotExistExports = exportedSymbolsList.filter( + (e: string) => !internallyExported.includes(e), ); console.log(missingSymbols); console.log(doesNotExistExports); From 1df6ec1fcaeba6e3ee865bd4893e5e3338f17460 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Tue, 17 Dec 2024 23:01:09 -0800 Subject: [PATCH 04/44] formatting fixed Signed-off-by: Prateek Kumar --- node/npm/glide/tests/ExportedSymbols.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/node/npm/glide/tests/ExportedSymbols.test.ts b/node/npm/glide/tests/ExportedSymbols.test.ts index 692b748f56..98e20217a5 100644 --- a/node/npm/glide/tests/ExportedSymbols.test.ts +++ b/node/npm/glide/tests/ExportedSymbols.test.ts @@ -15,7 +15,7 @@ describe("Exported Symbols test", () => { const filesWithNodeCode = await getFiles(implBuildFolder); console.log(filesWithNodeCode); - const internallyExported: any[] = []; + const internallyExported: string[] = []; for (const file of filesWithNodeCode) { const sourceCode = await f.readFile(file, "utf8"); @@ -30,7 +30,7 @@ describe("Exported Symbols test", () => { internallyExported.sort(); - let missingSymbols = internallyExported.filter( + const missingSymbols = internallyExported.filter( (e: string) => !exportedSymbolsList.includes(e), ); const doesNotExistExports = exportedSymbolsList.filter( @@ -71,6 +71,7 @@ async function getFiles(folderName: string): Promise { if (!file.name.endsWith(".d.ts")) { continue; } + filesWithNodeCode.push(folderName + file.name); } } @@ -81,8 +82,8 @@ async function getFiles(folderName: string): Promise { function visitRoot(root: ts.Node) { // (Root Level)->(Level 1) const children: ts.Node[] = root.getChildren(); - const resultList: string[] = []; + // (Root Level) -> (Level 1) -> Level 2. This is the level in the AST where all the exported symbols in a file are present. for (const node of children) { const nodeList: string[] = node From cb320e214c6536b0f2a06a6bbdedf1b1ff3f2c39 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Wed, 18 Dec 2024 10:05:34 -0800 Subject: [PATCH 05/44] Formatting fixed Signed-off-by: Prateek Kumar --- node/DEVELOPER.md | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/node/DEVELOPER.md b/node/DEVELOPER.md index 69be8ea15f..0d4a6ea26c 100644 --- a/node/DEVELOPER.md +++ b/node/DEVELOPER.md @@ -109,22 +109,23 @@ Before starting this step, make sure you've installed all software requirments. 6. Testing the GLIDE npm package: The `node/npm/glide` folder contains a package wrapper that re-exports all the native bindings. To build and test this package, follow these steps: - ```bash - Build node package: - co node - export nativeStr=darwin-x64; export scope=@valkey/; - envsubst < package.json.tmpl > "package.json" - npm run build - npm run build:release - In npm/glide package run the following commands: - cd ../npm/glide - export pkg_name=valkey-glide; export package_version=99.99.0; export scope=@valkey/; - envsubst < package.json.tmpl > "package.json" - npm run build - npm run build:test - npm run test - npm run test -- --testNamePattern="Exported symbols test" + ```bash + + Build node package: + co node + export nativeStr=darwin-x64; export scope=@valkey/; + envsubst < package.json.tmpl > "package.json" + npm run build + npm run build:release + In npm/glide package run the following commands: + cd ../npm/glide + export pkg_name=valkey-glide; export package_version=99.99.0; export scope=@valkey/; + envsubst < package.json.tmpl > "package.json" + npm run build + npm run build:test + npm run test + npm run test -- --testNamePattern="Exported symbols test" ``` - For a fast build, execute `npm run build`. This will perform a full, unoptimized build, which is suitable for developing tests. Keep in mind that performance is significantly affected in an unoptimized build, so it's required to build with the `build:release` or `build:benchmark` option when measuring performance. From 540180f5527fd17fd37450bb7ff0ca1f26f5e719 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Wed, 18 Dec 2024 10:07:25 -0800 Subject: [PATCH 06/44] Fix formatting Signed-off-by: Prateek Kumar --- node/DEVELOPER.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/node/DEVELOPER.md b/node/DEVELOPER.md index 0d4a6ea26c..23a146c9bd 100644 --- a/node/DEVELOPER.md +++ b/node/DEVELOPER.md @@ -107,17 +107,18 @@ Before starting this step, make sure you've installed all software requirments. 5. Integrating the built GLIDE package into your project: Add the package to your project using the folder path with the command `npm install /node`. -6. Testing the GLIDE npm package: +6. Testing the GLIDE npm package locally: The `node/npm/glide` folder contains a package wrapper that re-exports all the native bindings. To build and test this package, follow these steps: ```bash Build node package: - co node + cd node export nativeStr=darwin-x64; export scope=@valkey/; envsubst < package.json.tmpl > "package.json" npm run build npm run build:release + In npm/glide package run the following commands: cd ../npm/glide export pkg_name=valkey-glide; export package_version=99.99.0; export scope=@valkey/; From c2cbf13c62126910d6a62fa90d358853f5729cfb Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Wed, 18 Dec 2024 10:20:59 -0800 Subject: [PATCH 07/44] Formatting fixed Signed-off-by: Prateek Kumar --- node/npm/glide/jest.config.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/node/npm/glide/jest.config.js b/node/npm/glide/jest.config.js index c68ff7fa63..0a0eec9ef4 100644 --- a/node/npm/glide/jest.config.js +++ b/node/npm/glide/jest.config.js @@ -1,4 +1,3 @@ - /* eslint no-undef: off */ module.exports = { preset: "ts-jest", @@ -29,5 +28,5 @@ module.exports = { sort: "status", }, ], - ] + ], }; From 77c6c1cfd9ba9b92c8f44fcfe0d79a9f8521b0b3 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Wed, 18 Dec 2024 11:14:44 -0800 Subject: [PATCH 08/44] Node: Change log updated Signed-off-by: Prateek Kumar --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2c349d60f..97e15867b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ #### Changes - +* Node: Add node test for checking exported symbols ([#2835](https://github.com/valkey-io/valkey-glide/pull/2835)) * Go: Add SUNIONSTORE command ([#2805](https://github.com/valkey-io/valkey-glide/pull/2805) * Go: Add SUNION ([#2787](https://github.com/valkey-io/valkey-glide/pull/2787) * Java: bump `netty` version ([#2795](https://github.com/valkey-io/valkey-glide/pull/2795)) From 2d100ee5a33ccce5b5439e8c242f373befeb2847 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Tue, 14 Jan 2025 09:07:51 -0800 Subject: [PATCH 09/44] Node: Add npm install Signed-off-by: Prateek Kumar --- .github/workflows/npm-cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/npm-cd.yml b/.github/workflows/npm-cd.yml index 24497e83cc..55b82dd1cd 100644 --- a/.github/workflows/npm-cd.yml +++ b/.github/workflows/npm-cd.yml @@ -237,7 +237,6 @@ jobs: working-directory: ./node/npm/glide run: | export pkg_name=valkey-glide - echo "The workflow is: ${{env.EVENT_NAME}}" if ${{ env.EVENT_NAME == 'workflow_dispatch' }}; then R_VERSION="${{ env.INPUT_VERSION }}" @@ -253,6 +252,7 @@ jobs: cat package.json # Fix index.ts based on the scope variable sed -i "s|@scope/|${scope}|g" index.ts + npm install env: NPM_SCOPE: ${{ vars.NPM_SCOPE }} EVENT_NAME: ${{ github.event_name }} From 7d9ae5ac8aa1d08a8f17e4a11479c14c0f0e4b7b Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Tue, 14 Jan 2025 09:13:03 -0800 Subject: [PATCH 10/44] Node: update changelog.md Signed-off-by: Prateek Kumar --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97e15867b2..237caf9815 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,4 @@ #### Changes -* Node: Add node test for checking exported symbols ([#2835](https://github.com/valkey-io/valkey-glide/pull/2835)) * Go: Add SUNIONSTORE command ([#2805](https://github.com/valkey-io/valkey-glide/pull/2805) * Go: Add SUNION ([#2787](https://github.com/valkey-io/valkey-glide/pull/2787) * Java: bump `netty` version ([#2795](https://github.com/valkey-io/valkey-glide/pull/2795)) From d7c4254e3c247e77f885f2d2d2a359dd52a91443 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Tue, 14 Jan 2025 09:16:50 -0800 Subject: [PATCH 11/44] Node: Add changelog.md Signed-off-by: Prateek Kumar --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 237caf9815..b2c349d60f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ #### Changes + * Go: Add SUNIONSTORE command ([#2805](https://github.com/valkey-io/valkey-glide/pull/2805) * Go: Add SUNION ([#2787](https://github.com/valkey-io/valkey-glide/pull/2787) * Java: bump `netty` version ([#2795](https://github.com/valkey-io/valkey-glide/pull/2795)) From cd67e1c020229b463cc14e746f1fb265c4ef1c36 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Tue, 14 Jan 2025 10:15:19 -0800 Subject: [PATCH 12/44] Node: update workflow Signed-off-by: Prateek Kumar --- .github/workflows/node.yml | 4 +++- .github/workflows/npm-cd.yml | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index a91a853534..b0009ec752 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -221,7 +221,9 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: ${{ matrix.engine.version }} arch: ${{ matrix.host.ARCH }} - + - name: Run npm install in ./npm/glide package + run: npm install + working-directory: ./npm/glide - name: test run: npm test working-directory: ./node diff --git a/.github/workflows/npm-cd.yml b/.github/workflows/npm-cd.yml index 55b82dd1cd..39f96bc73e 100644 --- a/.github/workflows/npm-cd.yml +++ b/.github/workflows/npm-cd.yml @@ -252,7 +252,6 @@ jobs: cat package.json # Fix index.ts based on the scope variable sed -i "s|@scope/|${scope}|g" index.ts - npm install env: NPM_SCOPE: ${{ vars.NPM_SCOPE }} EVENT_NAME: ${{ github.event_name }} From def56198b9b5f4d3e0c01cf49fc04f96950c7580 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Tue, 14 Jan 2025 11:08:24 -0800 Subject: [PATCH 13/44] Node: Update workflow Signed-off-by: Prateek Kumar --- .github/workflows/node.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index b0009ec752..107f2029b7 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -223,7 +223,7 @@ jobs: arch: ${{ matrix.host.ARCH }} - name: Run npm install in ./npm/glide package run: npm install - working-directory: ./npm/glide + working-directory: ./node/npm/glide - name: test run: npm test working-directory: ./node From 522fed8750c0ebbd2c54cb5dddf845b64cb1c9c7 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Tue, 14 Jan 2025 11:14:37 -0800 Subject: [PATCH 14/44] Node: Update workflow Signed-off-by: Prateek Kumar --- .github/workflows/node.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 107f2029b7..e0137140c2 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -103,6 +103,10 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: ${{ matrix.engine.version }} + - name: Run npm install in ./npm/glide package + run: npm install + working-directory: ./node/npm/glide + - name: test run: npm test working-directory: ./node @@ -221,9 +225,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: ${{ matrix.engine.version }} arch: ${{ matrix.host.ARCH }} - - name: Run npm install in ./npm/glide package - run: npm install - working-directory: ./node/npm/glide + - name: test run: npm test working-directory: ./node From 5e0497017d1f3d43a55001f6127315534f608a96 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Tue, 21 Jan 2025 21:06:52 -0800 Subject: [PATCH 15/44] Node: npm/glide package json created Signed-off-by: Prateek Kumar --- .github/workflows/node.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index e0137140c2..0d905f6c23 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -104,7 +104,28 @@ jobs: engine-version: ${{ matrix.engine.version }} - name: Run npm install in ./npm/glide package - run: npm install + run: | + export pkg_name=valkey-glide + echo "The workflow is: ${{env.EVENT_NAME}}" + if ${{ env.EVENT_NAME == 'workflow_dispatch' }}; then + R_VERSION="${{ env.INPUT_VERSION }}" + else + R_VERSION=${GITHUB_REF:11} + fi + echo "RELEASE_VERSION=${R_VERSION}" >> $GITHUB_ENV + + export package_version=${R_VERSION} + export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi` + mv package.json package.json.tmpl + envsubst < package.json.tmpl > "package.json" + cat package.json + # Fix index.ts based on the scope variable + sed -i "s|@scope/|${scope}|g" index.ts + npm install + env: + NPM_SCOPE: ${{ vars.NPM_SCOPE }} + EVENT_NAME: ${{ github.event_name }} + INPUT_VERSION: ${{ github.event.inputs.version }} working-directory: ./node/npm/glide - name: test From 8c439483bfa57bcbdb2a9744e73fa54ae7be1586 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Tue, 21 Jan 2025 21:28:09 -0800 Subject: [PATCH 16/44] Node: npm/glide package json created Signed-off-by: Prateek Kumar --- node/npm/glide/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/npm/glide/index.ts b/node/npm/glide/index.ts index 87d08a5a3e..c06e94d010 100644 --- a/node/npm/glide/index.ts +++ b/node/npm/glide/index.ts @@ -11,8 +11,8 @@ let globalObject = global as unknown; /* eslint-disable @typescript-eslint/no-require-imports */ function loadNativeBinding() { - const scope = process.env.scope || "@scope"; - + let scope = process.env.scope || "@scope"; + console.log(process.env); let nativeBinding = []; let nativeStr = process.env.native_binding; From 5013e66d831bc882f30c7740f69b253f15cb3609 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Wed, 22 Jan 2025 00:15:47 -0800 Subject: [PATCH 17/44] Node: npm/glide package json created Signed-off-by: Prateek Kumar --- node/npm/glide/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/node/npm/glide/index.ts b/node/npm/glide/index.ts index c06e94d010..a39163c1d5 100644 --- a/node/npm/glide/index.ts +++ b/node/npm/glide/index.ts @@ -11,8 +11,9 @@ let globalObject = global as unknown; /* eslint-disable @typescript-eslint/no-require-imports */ function loadNativeBinding() { + console.log("@scope"); let scope = process.env.scope || "@scope"; - console.log(process.env); + //console.log(process.env); let nativeBinding = []; let nativeStr = process.env.native_binding; From f35176c502dd987dd81ec2fdad8982d912d1874f Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Wed, 22 Jan 2025 00:18:11 -0800 Subject: [PATCH 18/44] Node: npm/glide package json created Signed-off-by: Prateek Kumar --- .github/workflows/node.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 0d905f6c23..721f357876 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -120,7 +120,7 @@ jobs: envsubst < package.json.tmpl > "package.json" cat package.json # Fix index.ts based on the scope variable - sed -i "s|@scope/|${scope}|g" index.ts + sed -i "s/@scope/${scope}/g" index.ts npm install env: NPM_SCOPE: ${{ vars.NPM_SCOPE }} From ef4ca6e74d835c1fd4e1b57794108def89000315 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Wed, 22 Jan 2025 00:24:15 -0800 Subject: [PATCH 19/44] Node: npm/glide package json created Signed-off-by: Prateek Kumar --- .github/workflows/node.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 721f357876..15b3588283 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -120,7 +120,7 @@ jobs: envsubst < package.json.tmpl > "package.json" cat package.json # Fix index.ts based on the scope variable - sed -i "s/@scope/${scope}/g" index.ts + sed -i "s+@scope+${scope}+g" index.ts npm install env: NPM_SCOPE: ${{ vars.NPM_SCOPE }} From 6cdcddf55d07706c4ceed6bcebb4f3e943e2e431 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Wed, 22 Jan 2025 00:28:48 -0800 Subject: [PATCH 20/44] Node: npm/glide package json created Signed-off-by: Prateek Kumar --- .github/workflows/node.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 15b3588283..31574984c4 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -120,7 +120,7 @@ jobs: envsubst < package.json.tmpl > "package.json" cat package.json # Fix index.ts based on the scope variable - sed -i "s+@scope+${scope}+g" index.ts + sed -i 's/@scope/${scope}/g' index.ts npm install env: NPM_SCOPE: ${{ vars.NPM_SCOPE }} From 212e3676c6c14b05d5235b632d604f84697b3f98 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Wed, 22 Jan 2025 00:42:49 -0800 Subject: [PATCH 21/44] Node: npm/glide package json created Signed-off-by: Prateek Kumar --- .github/workflows/node.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 31574984c4..04fc69de0f 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -120,7 +120,7 @@ jobs: envsubst < package.json.tmpl > "package.json" cat package.json # Fix index.ts based on the scope variable - sed -i 's/@scope/${scope}/g' index.ts + sed -i 's/@scope/'+${scope}+'/g' index.ts npm install env: NPM_SCOPE: ${{ vars.NPM_SCOPE }} From 03f20232b4e258deae7da4d61fc5d3b0098ac3f8 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Wed, 22 Jan 2025 00:57:23 -0800 Subject: [PATCH 22/44] Node: npm/glide package json created Signed-off-by: Prateek Kumar --- .github/workflows/node.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 04fc69de0f..404bec495a 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -120,7 +120,8 @@ jobs: envsubst < package.json.tmpl > "package.json" cat package.json # Fix index.ts based on the scope variable - sed -i 's/@scope/'+${scope}+'/g' index.ts + sed -i'' "s/@scope/${scope}/g" index.ts + npm install env: NPM_SCOPE: ${{ vars.NPM_SCOPE }} From 35602e063d736410459da3cfac535a5d4de0fab6 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Wed, 22 Jan 2025 01:04:44 -0800 Subject: [PATCH 23/44] Node: npm/glide package json created Signed-off-by: Prateek Kumar --- .github/workflows/node.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 404bec495a..09d0f180ac 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -120,7 +120,7 @@ jobs: envsubst < package.json.tmpl > "package.json" cat package.json # Fix index.ts based on the scope variable - sed -i'' "s/@scope/${scope}/g" index.ts + sed -i "s/\@scope/${scope}/g" index.ts npm install env: From ebd30f4956705b5295ba9bdf8e44988200263c29 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Wed, 22 Jan 2025 01:11:15 -0800 Subject: [PATCH 24/44] Node: npm/glide package json created Signed-off-by: Prateek Kumar --- .github/workflows/node.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 09d0f180ac..fc8f47cb50 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -120,8 +120,8 @@ jobs: envsubst < package.json.tmpl > "package.json" cat package.json # Fix index.ts based on the scope variable - sed -i "s/\@scope/${scope}/g" index.ts - + # sed -i "s/\@scope/${scope}/g" index.ts + sed -i "s|@scope/|${scope}|g" index.ts npm install env: NPM_SCOPE: ${{ vars.NPM_SCOPE }} From a8f4a129198ac03e3bcfe5ac40c50aaf036f5594 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Wed, 22 Jan 2025 10:08:05 -0800 Subject: [PATCH 25/44] Node: npm/glide package json created Signed-off-by: Prateek Kumar --- node/npm/glide/index.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/node/npm/glide/index.ts b/node/npm/glide/index.ts index a39163c1d5..dfd8487c46 100644 --- a/node/npm/glide/index.ts +++ b/node/npm/glide/index.ts @@ -11,12 +11,10 @@ let globalObject = global as unknown; /* eslint-disable @typescript-eslint/no-require-imports */ function loadNativeBinding() { - console.log("@scope"); let scope = process.env.scope || "@scope"; - //console.log(process.env); let nativeBinding = []; let nativeStr = process.env.native_binding; - + console.log(nativeStr); if (nativeStr == undefined) { switch (platform) { case "linux": From d8fc4b9a66b1f8e606f1a7a42109e7e39f3bc79b Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Wed, 22 Jan 2025 12:00:51 -0800 Subject: [PATCH 26/44] Node: npm/glide package json created Signed-off-by: Prateek Kumar --- node/npm/glide/index.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/node/npm/glide/index.ts b/node/npm/glide/index.ts index dfd8487c46..ac8693b44e 100644 --- a/node/npm/glide/index.ts +++ b/node/npm/glide/index.ts @@ -15,6 +15,9 @@ function loadNativeBinding() { let nativeBinding = []; let nativeStr = process.env.native_binding; console.log(nativeStr); + if (scope == "@scope") { + scope = "@valkey/" + } if (nativeStr == undefined) { switch (platform) { case "linux": From 4d92a5d883732882b401e2369f00af3ee4a4ae9c Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Fri, 24 Jan 2025 02:38:10 -0800 Subject: [PATCH 27/44] Node: update workflow Signed-off-by: Prateek Kumar --- .github/workflows/node.yml | 4 +++- node/npm/glide/{package.json.tmpl => package.json.temp} | 0 2 files changed, 3 insertions(+), 1 deletion(-) rename node/npm/glide/{package.json.tmpl => package.json.temp} (100%) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index fc8f47cb50..a62008ad38 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -116,8 +116,10 @@ jobs: export package_version=${R_VERSION} export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi` - mv package.json package.json.tmpl envsubst < package.json.tmpl > "package.json" + + # mv package.json package.json.tmpl + # envsubst < package.json.tmpl > "package.json" cat package.json # Fix index.ts based on the scope variable # sed -i "s/\@scope/${scope}/g" index.ts diff --git a/node/npm/glide/package.json.tmpl b/node/npm/glide/package.json.temp similarity index 100% rename from node/npm/glide/package.json.tmpl rename to node/npm/glide/package.json.temp From a2bf0f1e18b119590d8d1ba95987370f8729f852 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Fri, 24 Jan 2025 02:40:21 -0800 Subject: [PATCH 28/44] Node: update workflow Signed-off-by: Prateek Kumar --- .github/workflows/node.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index a62008ad38..49cd859d3c 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -116,7 +116,7 @@ jobs: export package_version=${R_VERSION} export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi` - envsubst < package.json.tmpl > "package.json" + envsubst < package.json.temp > "package.json" # mv package.json package.json.tmpl # envsubst < package.json.tmpl > "package.json" From 8cbfbe9dbdfcdaba02ac4d6d9c073f1ea37339c6 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Fri, 24 Jan 2025 02:48:27 -0800 Subject: [PATCH 29/44] Node: update workflow Signed-off-by: Prateek Kumar --- node/npm/glide/package.json.temp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/npm/glide/package.json.temp b/node/npm/glide/package.json.temp index bc5ae26eb7..fb7538e3bf 100644 --- a/node/npm/glide/package.json.temp +++ b/node/npm/glide/package.json.temp @@ -45,7 +45,7 @@ "@typescript-eslint/parser": "^5.48.0", "eslint": "^8.31.0", "typescript": "^4.9.4", - "${scope}valkey-glide-${nativeStr}": "../.." + "${scope}valkey-glide-linux-x64": "../.." }, "optionalDependencies": { "${scope}valkey-glide-darwin-arm64": "${package_version}", From 23d4df33e9172983d53ab64aedc270e5b7ce7d20 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Mon, 27 Jan 2025 09:27:20 -0800 Subject: [PATCH 30/44] Update DEVELOPER.md and README.md files --- node/DEVELOPER.md | 66 ++++++++++++++++------------------------------- node/README.md | 8 +++--- 2 files changed, 26 insertions(+), 48 deletions(-) diff --git a/node/DEVELOPER.md b/node/DEVELOPER.md index 23a146c9bd..9568e3be82 100644 --- a/node/DEVELOPER.md +++ b/node/DEVELOPER.md @@ -16,14 +16,14 @@ Software Dependencies If your Nodejs version is below the supported version specified in the client's [documentation](https://github.com/valkey-io/valkey-glide/blob/main/node/README.md#nodejs-supported-version), you can upgrade it using [NVM](https://github.com/nvm-sh/nvm?tab=readme-ov-file#install--update-script). -- npm -- git -- GCC -- pkg-config -- protoc (protobuf compiler) -- openssl -- openssl-dev -- rustup +- npm +- git +- GCC +- pkg-config +- protoc (protobuf compiler) +- openssl +- openssl-dev +- rustup **Dependencies installation for Ubuntu** @@ -107,38 +107,16 @@ Before starting this step, make sure you've installed all software requirments. 5. Integrating the built GLIDE package into your project: Add the package to your project using the folder path with the command `npm install /node`. -6. Testing the GLIDE npm package locally: - The `node/npm/glide` folder contains a package wrapper that re-exports all the native bindings. To build and test this package, follow these steps: - - ```bash - - Build node package: - cd node - export nativeStr=darwin-x64; export scope=@valkey/; - envsubst < package.json.tmpl > "package.json" - npm run build - npm run build:release - - In npm/glide package run the following commands: - cd ../npm/glide - export pkg_name=valkey-glide; export package_version=99.99.0; export scope=@valkey/; - envsubst < package.json.tmpl > "package.json" - npm run build - npm run build:test - npm run test - npm run test -- --testNamePattern="Exported symbols test" - ``` - -- For a fast build, execute `npm run build`. This will perform a full, unoptimized build, which is suitable for developing tests. Keep in mind that performance is significantly affected in an unoptimized build, so it's required to build with the `build:release` or `build:benchmark` option when measuring performance. -- If your modifications are limited to the TypeScript code, run `npm run build-external` to build the external package without rebuilding the internal package. -- If your modifications are limited to the Rust code, execute `npm run build-internal` to build the internal package and generate TypeScript code. -- To generate Node's protobuf files, execute `npm run build-protobuf`. Keep in mind that protobuf files are generated as part of full builds (e.g., `build`, `build:release`, etc.). +- For a fast build, execute `npm run build`. This will perform a full, unoptimized build, which is suitable for developing tests. Keep in mind that performance is significantly affected in an unoptimized build, so it's required to build with the `build:release` or `build:benchmark` option when measuring performance. +- If your modifications are limited to the TypeScript code, run `npm run build-external` to build the external package without rebuilding the internal package. +- If your modifications are limited to the Rust code, execute `npm run build-internal` to build the internal package and generate TypeScript code. +- To generate Node's protobuf files, execute `npm run build-protobuf`. Keep in mind that protobuf files are generated as part of full builds (e.g., `build`, `build:release`, etc.). > Note: Once building completed, you'll find the compiled JavaScript code in the `node/build-ts` folder. ### Troubleshooting -- If the build fails after running `npx tsc` because `glide-rs` isn't found, check if your npm version is in the range 9.0.0-9.4.1, and if so, upgrade it. 9.4.2 contains a fix to a change introduced in 9.0.0 that is required in order to build the library. +- If the build fails after running `npx tsc` because `glide-rs` isn't found, check if your npm version is in the range 9.0.0-9.4.1, and if so, upgrade it. 9.4.2 contains a fix to a change introduced in 9.0.0 that is required in order to build the library. ### Test @@ -224,13 +202,13 @@ Development on the Node wrapper may involve changes in either the TypeScript or **TypeScript:** -- ESLint -- Prettier +- ESLint +- Prettier **Rust:** -- clippy -- fmt +- clippy +- fmt #### Running the linters @@ -253,8 +231,8 @@ Development on the Node wrapper may involve changes in either the TypeScript or ### Recommended extensions for VS Code -- [Prettier - Code formatter](https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode) - JavaScript / TypeScript formatter. -- [ESLint](https://marketplace.visualstudio.com/items?itemName=dbaeumer.vscode-eslint) - linter. -- [Jest Runner](https://marketplace.visualstudio.com/items?itemName=firsttris.vscode-jest-runner) - in-editor test runner. -- [Jest Test Explorer](https://marketplace.visualstudio.com/items?itemName=kavod-io.vscode-jest-test-adapter) - adapter to the VSCode testing UI. -- [rust-analyzer](https://marketplace.visualstudio.com/items?itemName=rust-lang.rust-analyzer) - Rust language support for VSCode. +- [Prettier - Code formatter](https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode) - JavaScript / TypeScript formatter. +- [ESLint](https://marketplace.visualstudio.com/items?itemName=dbaeumer.vscode-eslint) - linter. +- [Jest Runner](https://marketplace.visualstudio.com/items?itemName=firsttris.vscode-jest-runner) - in-editor test runner. +- [Jest Test Explorer](https://marketplace.visualstudio.com/items?itemName=kavod-io.vscode-jest-test-adapter) - adapter to the VSCode testing UI. +- [rust-analyzer](https://marketplace.visualstudio.com/items?itemName=rust-lang.rust-analyzer) - Rust language support for VSCode. diff --git a/node/README.md b/node/README.md index 299499d202..eeaf4e4a60 100644 --- a/node/README.md +++ b/node/README.md @@ -14,16 +14,16 @@ The release of Valkey GLIDE was tested on the following platforms: Linux: -- Ubuntu 22.04.1 (x86_64 and aarch64) -- Amazon Linux 2023 (AL2023) (x86_64) +- Ubuntu 22.04.1 (x86_64 and aarch64) +- Amazon Linux 2023 (AL2023) (x86_64) macOS: -- macOS 14.7 (Apple silicon/aarch_64) +- macOS 14.7 (Apple silicon/aarch_64) Alpine: -- node:alpine (default on aarch64 and x86_64) +- node:alpine (default on aarch64 and x86_64) ## NodeJS supported version From 71a39393f0259111b66bebc7db46bd1217962106 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Mon, 27 Jan 2025 09:28:32 -0800 Subject: [PATCH 31/44] Fix formatting Signed-off-by: Prateek Kumar --- .github/workflows/npm-cd.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/npm-cd.yml b/.github/workflows/npm-cd.yml index 8eaad532b9..63fe2887de 100644 --- a/.github/workflows/npm-cd.yml +++ b/.github/workflows/npm-cd.yml @@ -237,6 +237,7 @@ jobs: working-directory: ./node/npm/glide run: | export pkg_name=valkey-glide + echo "The workflow is: ${{env.EVENT_NAME}}" if ${{ env.EVENT_NAME == 'workflow_dispatch' }}; then R_VERSION="${{ env.INPUT_VERSION }}" From 4c8ca4b5d7fb1eb51c31b90c6ffd4eee38660800 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Mon, 27 Jan 2025 09:29:02 -0800 Subject: [PATCH 32/44] Fix formatting Signed-off-by: Prateek Kumar --- .github/workflows/npm-cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/npm-cd.yml b/.github/workflows/npm-cd.yml index 63fe2887de..3e5f673a4c 100644 --- a/.github/workflows/npm-cd.yml +++ b/.github/workflows/npm-cd.yml @@ -237,7 +237,7 @@ jobs: working-directory: ./node/npm/glide run: | export pkg_name=valkey-glide - + echo "The workflow is: ${{env.EVENT_NAME}}" if ${{ env.EVENT_NAME == 'workflow_dispatch' }}; then R_VERSION="${{ env.INPUT_VERSION }}" From ffb231ee4fddb847c4ea6a1055d27cc873d9e96c Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Mon, 27 Jan 2025 09:30:57 -0800 Subject: [PATCH 33/44] package.json.tmpl file removed Signed-off-by: Prateek Kumar --- node/package.json.tmpl | 97 ------------------------------------------ 1 file changed, 97 deletions(-) delete mode 100644 node/package.json.tmpl diff --git a/node/package.json.tmpl b/node/package.json.tmpl deleted file mode 100644 index 6a8b50b96c..0000000000 --- a/node/package.json.tmpl +++ /dev/null @@ -1,97 +0,0 @@ -{ - "name": "@valkey/valkey-glide-impl", - "description": "General Language Independent Driver for the Enterprise (GLIDE) for Valkey", - "main": "build-ts/index.js", - "module": "build-ts/index.js", - "types": "./build-ts/index.d.ts", - "type": "commonjs", - "repository": { - "type": "git", - "url": "git+https://github.com/valkey-io/valkey-glide.git" - }, - "homepage": "https://github.com/valkey-io/valkey-glide#readme", - "dependencies": { - "glide-rs": "file:rust-client", - "long": "^5.2.3", - "npmignore": "^0.3.1", - "protobufjs": "^7.4.0" - }, - "bundleDependencies": [ - "glide-rs" - ], - "scripts": { - "build": "npm run prereq && npm run build-internal && npm run build-protobuf && npm run build-external", - "build:release": "npm run build-internal:release && npm run build-protobuf && npm run build-external:release", - "build:benchmark": "npm run build-internal:benchmark && npm run build-protobuf && npm run build-external", - "build-internal": "cd rust-client && npm run build", - "build-internal:release": "cd rust-client && npm run build:release", - "build-internal:benchmark": "cd rust-client && npm run build:benchmark", - "build-external": "rm -rf build-ts && tsc", - "build-external:release": "rm -rf build-ts && tsc --stripInternal", - "build-protobuf": "npm run compile-protobuf-files && npm run fix-protobuf-file", - "compile-protobuf-files": "cd src && pbjs -t static-module -o ProtobufMessage.js ../../glide-core/src/protobuf/*.proto && pbts -o ProtobufMessage.d.ts ProtobufMessage.js", - "clean": "rm -rf build-ts rust-client/target docs glide-logs rust-client/glide-rs.*.node rust-client/index.* src/ProtobufMessage.*", - "fix-protobuf-file": "replace 'this\\.encode\\(message, writer\\)\\.ldelim' 'this.encode(message, writer && writer.len ? writer.fork() : writer).ldelim' src/ProtobufMessage.js", - "test": "npm run build-test-utils && jest --verbose --testPathIgnorePatterns='ServerModules'", - "test-dbg": "npm run build-test-utils && jest --runInBand", - "test-minimum": "npm run build-test-utils && jest --verbose --testNamePattern='^(.(?!(GlideJson|GlideFt|pubsub|kill)))*$'", - "test-modules": "npm run build-test-utils && jest --verbose --testNamePattern='(GlideJson|GlideFt)'", - "build-test-utils": "cd ../utils && npm i && npm run build", - "lint:fix": "npm run install-linting && npx eslint -c ../eslint.config.mjs --fix && npm run prettier:format", - "lint": "npm run install-linting && npx eslint -c ../eslint.config.mjs && npm run prettier:check:ci", - "install-linting": "cd ../ & npm install", - "prepack": "npmignore --auto", - "prereq": "npm install", - "prettier:check:ci": "npx prettier --check . --ignore-unknown '!**/*.{js,d.ts}'", - "prettier:format": "npx prettier --write . --ignore-unknown '!**/*.{js,d.ts}'" - }, - "devDependencies": { - "@jest/globals": "^29.7.0", - "@types/jest": "^29.5.14", - "@types/minimist": "^1.2.5", - "@types/redis-server": "^1.2.2", - "@types/semver": "^7.5.8", - "@types/uuid": "^10.0.0", - "find-free-port": "^2.0.0", - "jest": "^29.7.0", - "jest-html-reporter": "^3.10.2", - "protobufjs-cli": "^1.1.3", - "redis-server": "^1.2.2", - "replace": "^1.2.2", - "semver": "^7.6.3", - "ts-jest": "^29.2.5", - "ts-node": "^10.9.2", - "typescript": "^5.6.3", - "uuid": "^11.0.3" - }, - "author": "Valkey GLIDE Maintainers", - "license": "Apache-2.0", - "publishConfig": { - "${registry_scope}registry": "https://registry.npmjs.org/", - "ignore": [ - "src/**", - "tests/", - "rust-client/**", - "!build-ts/**", - ".prettierignore", - "jest.config.js", - "hybrid-node-tests/**", - "docs/", - "DEVELOPER.md", - ".ort.yml", - "tsconfig.json", - "THIRD_PARTY_LICENSES_NODE" - ] - }, - "//": [ - "The fields below have been commented out and are only necessary for publishing the package." - ], - "///cpu": [ - "${node_arch}" - ], - "///os": [ - "${node_os}" - ], - "///name": "${scope}valkey-glide-${nativeStr}", - "///version": "${package_version}" -} From 42fa269d33fb84d35e8c2db8239b00a003aeaeaa Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Mon, 27 Jan 2025 09:34:43 -0800 Subject: [PATCH 34/44] Updated env variables Signed-off-by: Prateek Kumar --- .github/workflows/node.yml | 6 ++---- node/npm/glide/package.json.temp | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 49cd859d3c..5b136ab672 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -113,13 +113,11 @@ jobs: R_VERSION=${GITHUB_REF:11} fi echo "RELEASE_VERSION=${R_VERSION}" >> $GITHUB_ENV - + export arch = ${{ matrix.host.ARCH }} + export platform = ${{ matrix.host.TARGET }} export package_version=${R_VERSION} export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi` envsubst < package.json.temp > "package.json" - - # mv package.json package.json.tmpl - # envsubst < package.json.tmpl > "package.json" cat package.json # Fix index.ts based on the scope variable # sed -i "s/\@scope/${scope}/g" index.ts diff --git a/node/npm/glide/package.json.temp b/node/npm/glide/package.json.temp index fb7538e3bf..8e7735f24f 100644 --- a/node/npm/glide/package.json.temp +++ b/node/npm/glide/package.json.temp @@ -45,7 +45,7 @@ "@typescript-eslint/parser": "^5.48.0", "eslint": "^8.31.0", "typescript": "^4.9.4", - "${scope}valkey-glide-linux-x64": "../.." + "${scope}valkey-glide-${platform}-${arch}": "../.." }, "optionalDependencies": { "${scope}valkey-glide-darwin-arm64": "${package_version}", From 481ec1d640ddb22de0c7087897de31148d7b7652 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Mon, 27 Jan 2025 09:58:46 -0800 Subject: [PATCH 35/44] Node update formatting Signed-off-by: Prateek Kumar --- .github/workflows/node.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 5b136ab672..21dad48477 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -113,8 +113,8 @@ jobs: R_VERSION=${GITHUB_REF:11} fi echo "RELEASE_VERSION=${R_VERSION}" >> $GITHUB_ENV - export arch = ${{ matrix.host.ARCH }} - export platform = ${{ matrix.host.TARGET }} + export arch="${{ matrix.host.ARCH }}"" + export platform="${{ matrix.host.TARGET }}" export package_version=${R_VERSION} export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi` envsubst < package.json.temp > "package.json" From 1365fb6d3f2a6b84c41fd7605c909e20ab9db341 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Mon, 27 Jan 2025 10:11:53 -0800 Subject: [PATCH 36/44] Node update formatting Signed-off-by: Prateek Kumar --- .github/workflows/node.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 21dad48477..50484da06b 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -113,7 +113,7 @@ jobs: R_VERSION=${GITHUB_REF:11} fi echo "RELEASE_VERSION=${R_VERSION}" >> $GITHUB_ENV - export arch="${{ matrix.host.ARCH }}"" + export arch="${{ matrix.host.ARCH }}" export platform="${{ matrix.host.TARGET }}" export package_version=${R_VERSION} export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi` From 242103e6132ee2a82faa8e40f42747a46c39b40f Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Mon, 27 Jan 2025 14:28:13 -0800 Subject: [PATCH 37/44] Node update formatting Signed-off-by: Prateek Kumar --- .github/workflows/node.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 50484da06b..6fa2a17342 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -114,7 +114,7 @@ jobs: fi echo "RELEASE_VERSION=${R_VERSION}" >> $GITHUB_ENV export arch="${{ matrix.host.ARCH }}" - export platform="${{ matrix.host.TARGET }}" + export named_os="${{ matrix.host.NAMED_OS }}" export package_version=${R_VERSION} export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi` envsubst < package.json.temp > "package.json" From 5828ee93fe502341dd5db578a7fede9b5314890e Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Mon, 27 Jan 2025 14:34:12 -0800 Subject: [PATCH 38/44] Node update formatting Signed-off-by: Prateek Kumar --- node/npm/glide/package.json.temp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/npm/glide/package.json.temp b/node/npm/glide/package.json.temp index 8e7735f24f..a20f82e026 100644 --- a/node/npm/glide/package.json.temp +++ b/node/npm/glide/package.json.temp @@ -45,7 +45,7 @@ "@typescript-eslint/parser": "^5.48.0", "eslint": "^8.31.0", "typescript": "^4.9.4", - "${scope}valkey-glide-${platform}-${arch}": "../.." + "${scope}valkey-glide-${named_os}-${arch}": "../.." }, "optionalDependencies": { "${scope}valkey-glide-darwin-arm64": "${package_version}", From 297db5b7c8fd6032b6293406b2b3c2dd97e51c16 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Tue, 28 Jan 2025 19:57:06 -0800 Subject: [PATCH 39/44] Node: index.ts updated Signed-off-by: Prateek Kumar --- node/npm/glide/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/node/npm/glide/index.ts b/node/npm/glide/index.ts index ac8693b44e..598557da5b 100644 --- a/node/npm/glide/index.ts +++ b/node/npm/glide/index.ts @@ -14,7 +14,6 @@ function loadNativeBinding() { let scope = process.env.scope || "@scope"; let nativeBinding = []; let nativeStr = process.env.native_binding; - console.log(nativeStr); if (scope == "@scope") { scope = "@valkey/" } From 390e9ad3e003ffb0c37e736e7a2bdeda6075b964 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Tue, 28 Jan 2025 20:01:02 -0800 Subject: [PATCH 40/44] Node: index.ts updated Signed-off-by: Prateek Kumar --- node/npm/glide/tests/ExportedSymbols.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/npm/glide/tests/ExportedSymbols.test.ts b/node/npm/glide/tests/ExportedSymbols.test.ts index 98e20217a5..7ea08a4e68 100644 --- a/node/npm/glide/tests/ExportedSymbols.test.ts +++ b/node/npm/glide/tests/ExportedSymbols.test.ts @@ -2,7 +2,7 @@ import { it } from "@jest/globals"; import * as f from "fs/promises"; import { describe } from "node:test"; import * as ts from "typescript"; -import * as glideApi from "../"; +import * as glideApi from "../"; //ESM convention, describe("Exported Symbols test", () => { it("check excluded symbols are not exported", async () => { From caca26678c1aec6a8f94364e6323f14618af9d8a Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Tue, 28 Jan 2025 20:16:20 -0800 Subject: [PATCH 41/44] Node: Review comments updated Signed-off-by: Prateek Kumar --- node/npm/glide/index.ts | 11 +++++------ node/npm/glide/package.json.temp | 1 - 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/node/npm/glide/index.ts b/node/npm/glide/index.ts index 598557da5b..17bde3af62 100644 --- a/node/npm/glide/index.ts +++ b/node/npm/glide/index.ts @@ -11,12 +11,7 @@ let globalObject = global as unknown; /* eslint-disable @typescript-eslint/no-require-imports */ function loadNativeBinding() { - let scope = process.env.scope || "@scope"; - let nativeBinding = []; let nativeStr = process.env.native_binding; - if (scope == "@scope") { - scope = "@valkey/" - } if (nativeStr == undefined) { switch (platform) { case "linux": @@ -74,7 +69,11 @@ function loadNativeBinding() { } } - nativeBinding = require(`${scope}valkey-glide-${nativeStr}`); + let scope = process.env.scope || "@scope"; + if (scope == "@scope") { + scope = "@valkey/" + } + const nativeBinding = require(`${scope}valkey-glide-${nativeStr}`); if (!nativeBinding) { throw new Error(`Failed to load native binding`); diff --git a/node/npm/glide/package.json.temp b/node/npm/glide/package.json.temp index a20f82e026..d38d060ed6 100644 --- a/node/npm/glide/package.json.temp +++ b/node/npm/glide/package.json.temp @@ -49,7 +49,6 @@ }, "optionalDependencies": { "${scope}valkey-glide-darwin-arm64": "${package_version}", - "${scope}valkey-glide-darwin-x64": "${package_version}", "${scope}valkey-glide-linux-arm64": "${package_version}", "${scope}valkey-glide-linux-x64": "${package_version}", "${scope}valkey-glide-linux-musl-arm64": "${package_version}", From 1242f693f61e216863ff438bce9c71a07438a3fb Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Wed, 29 Jan 2025 09:43:39 -0800 Subject: [PATCH 42/44] Node: fix test case Signed-off-by: Prateek Kumar --- node/npm/glide/tests/ExportedSymbols.test.ts | 30 ++++++++++++++------ 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/node/npm/glide/tests/ExportedSymbols.test.ts b/node/npm/glide/tests/ExportedSymbols.test.ts index 7ea08a4e68..001623beb4 100644 --- a/node/npm/glide/tests/ExportedSymbols.test.ts +++ b/node/npm/glide/tests/ExportedSymbols.test.ts @@ -7,11 +7,9 @@ import * as glideApi from "../"; //ESM convention, describe("Exported Symbols test", () => { it("check excluded symbols are not exported", async () => { // Check exported symbols for valkey glide package - const exportedSymbolsList = Object.keys(glideApi).sort(); // exportedList + const exportedSymbolsList = Object.keys(glideApi).sort(); // exportedList from the npm/glide package. const implBuildFolder = "./build-ts/"; - // const rustClientBuildFolder = './node_modules/@valkey/valkey-glide-impl/rust-client/'; - // const filesWithNodeCode = [...await getFiles(implBuildFolder), ...await getFiles(rustClientBuildFolder)]; const filesWithNodeCode = await getFiles(implBuildFolder); console.log(filesWithNodeCode); @@ -30,15 +28,31 @@ describe("Exported Symbols test", () => { internallyExported.sort(); + const skippedListForExports: string[] = ['AdvancedBaseClientConfiguration', + 'ClusterScanOptions', 'GlideMultiJson' + ]; const missingSymbols = internallyExported.filter( - (e: string) => !exportedSymbolsList.includes(e), + (e: string) => (!exportedSymbolsList.includes(e) && !skippedListForExports.includes(e)), ); + + const glideRsKeyWords: string[] = ['ClusterScanCursor', 'Script', 'createLeakedArray', + 'createLeakedAttribute', 'createLeakedBigint', 'createLeakedDouble', 'createLeakedMap', + 'createLeakedString', 'default']; const doesNotExistExports = exportedSymbolsList.filter( - (e: string) => !internallyExported.includes(e), + (e: string) => (!internallyExported.includes(e) && !glideRsKeyWords.includes(e)), ); - console.log(missingSymbols); - console.log(doesNotExistExports); - expect(exportedSymbolsList).toEqual(internallyExported); + if (missingSymbols.length > 0) { + console.log('The following symbols are exported from npm/glide package but missing ' + + 'from the internal node package export. These symbols might be from glide-rs package'); + console.log(missingSymbols); + } + expect(missingSymbols.length).toBe(0); + + if (doesNotExistExports.length > 0) { + console.log("Symbols that might be missed from the npm/glide package export:") + console.log(doesNotExistExports); + } + expect(doesNotExistExports.length).toBe(0); }); }); From 8fe390ed1e61f9d63c6272ae62f809e6aa68bbe9 Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Wed, 29 Jan 2025 10:00:08 -0800 Subject: [PATCH 43/44] Node: fix formatting Signed-off-by: Prateek Kumar --- node/npm/glide/index.ts | 3 +++ node/npm/glide/tests/ExportedSymbols.test.ts | 3 +++ 2 files changed, 6 insertions(+) diff --git a/node/npm/glide/index.ts b/node/npm/glide/index.ts index 17bde3af62..2dda300fb3 100644 --- a/node/npm/glide/index.ts +++ b/node/npm/glide/index.ts @@ -12,6 +12,7 @@ let globalObject = global as unknown; /* eslint-disable @typescript-eslint/no-require-imports */ function loadNativeBinding() { let nativeStr = process.env.native_binding; + if (nativeStr == undefined) { switch (platform) { case "linux": @@ -70,9 +71,11 @@ function loadNativeBinding() { } let scope = process.env.scope || "@scope"; + if (scope == "@scope") { scope = "@valkey/" } + const nativeBinding = require(`${scope}valkey-glide-${nativeStr}`); if (!nativeBinding) { diff --git a/node/npm/glide/tests/ExportedSymbols.test.ts b/node/npm/glide/tests/ExportedSymbols.test.ts index 001623beb4..948a2afe43 100644 --- a/node/npm/glide/tests/ExportedSymbols.test.ts +++ b/node/npm/glide/tests/ExportedSymbols.test.ts @@ -41,17 +41,20 @@ describe("Exported Symbols test", () => { const doesNotExistExports = exportedSymbolsList.filter( (e: string) => (!internallyExported.includes(e) && !glideRsKeyWords.includes(e)), ); + if (missingSymbols.length > 0) { console.log('The following symbols are exported from npm/glide package but missing ' + 'from the internal node package export. These symbols might be from glide-rs package'); console.log(missingSymbols); } + expect(missingSymbols.length).toBe(0); if (doesNotExistExports.length > 0) { console.log("Symbols that might be missed from the npm/glide package export:") console.log(doesNotExistExports); } + expect(doesNotExistExports.length).toBe(0); }); }); From 0f07070fd5d93b4060dc0c1e9406f5a28182095b Mon Sep 17 00:00:00 2001 From: Prateek Kumar Date: Wed, 29 Jan 2025 10:08:51 -0800 Subject: [PATCH 44/44] Node: Fix formatting Signed-off-by: Prateek Kumar --- node/DEVELOPER.md | 44 ++++++++++---------- node/README.md | 8 ++-- node/npm/glide/index.ts | 2 +- node/npm/glide/tests/ExportedSymbols.test.ts | 39 ++++++++++++----- 4 files changed, 55 insertions(+), 38 deletions(-) diff --git a/node/DEVELOPER.md b/node/DEVELOPER.md index 9568e3be82..34df6e41d0 100644 --- a/node/DEVELOPER.md +++ b/node/DEVELOPER.md @@ -16,14 +16,14 @@ Software Dependencies If your Nodejs version is below the supported version specified in the client's [documentation](https://github.com/valkey-io/valkey-glide/blob/main/node/README.md#nodejs-supported-version), you can upgrade it using [NVM](https://github.com/nvm-sh/nvm?tab=readme-ov-file#install--update-script). -- npm -- git -- GCC -- pkg-config -- protoc (protobuf compiler) -- openssl -- openssl-dev -- rustup +- npm +- git +- GCC +- pkg-config +- protoc (protobuf compiler) +- openssl +- openssl-dev +- rustup **Dependencies installation for Ubuntu** @@ -107,16 +107,16 @@ Before starting this step, make sure you've installed all software requirments. 5. Integrating the built GLIDE package into your project: Add the package to your project using the folder path with the command `npm install /node`. -- For a fast build, execute `npm run build`. This will perform a full, unoptimized build, which is suitable for developing tests. Keep in mind that performance is significantly affected in an unoptimized build, so it's required to build with the `build:release` or `build:benchmark` option when measuring performance. -- If your modifications are limited to the TypeScript code, run `npm run build-external` to build the external package without rebuilding the internal package. -- If your modifications are limited to the Rust code, execute `npm run build-internal` to build the internal package and generate TypeScript code. -- To generate Node's protobuf files, execute `npm run build-protobuf`. Keep in mind that protobuf files are generated as part of full builds (e.g., `build`, `build:release`, etc.). +- For a fast build, execute `npm run build`. This will perform a full, unoptimized build, which is suitable for developing tests. Keep in mind that performance is significantly affected in an unoptimized build, so it's required to build with the `build:release` or `build:benchmark` option when measuring performance. +- If your modifications are limited to the TypeScript code, run `npm run build-external` to build the external package without rebuilding the internal package. +- If your modifications are limited to the Rust code, execute `npm run build-internal` to build the internal package and generate TypeScript code. +- To generate Node's protobuf files, execute `npm run build-protobuf`. Keep in mind that protobuf files are generated as part of full builds (e.g., `build`, `build:release`, etc.). > Note: Once building completed, you'll find the compiled JavaScript code in the `node/build-ts` folder. ### Troubleshooting -- If the build fails after running `npx tsc` because `glide-rs` isn't found, check if your npm version is in the range 9.0.0-9.4.1, and if so, upgrade it. 9.4.2 contains a fix to a change introduced in 9.0.0 that is required in order to build the library. +- If the build fails after running `npx tsc` because `glide-rs` isn't found, check if your npm version is in the range 9.0.0-9.4.1, and if so, upgrade it. 9.4.2 contains a fix to a change introduced in 9.0.0 that is required in order to build the library. ### Test @@ -202,13 +202,13 @@ Development on the Node wrapper may involve changes in either the TypeScript or **TypeScript:** -- ESLint -- Prettier +- ESLint +- Prettier **Rust:** -- clippy -- fmt +- clippy +- fmt #### Running the linters @@ -231,8 +231,8 @@ Development on the Node wrapper may involve changes in either the TypeScript or ### Recommended extensions for VS Code -- [Prettier - Code formatter](https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode) - JavaScript / TypeScript formatter. -- [ESLint](https://marketplace.visualstudio.com/items?itemName=dbaeumer.vscode-eslint) - linter. -- [Jest Runner](https://marketplace.visualstudio.com/items?itemName=firsttris.vscode-jest-runner) - in-editor test runner. -- [Jest Test Explorer](https://marketplace.visualstudio.com/items?itemName=kavod-io.vscode-jest-test-adapter) - adapter to the VSCode testing UI. -- [rust-analyzer](https://marketplace.visualstudio.com/items?itemName=rust-lang.rust-analyzer) - Rust language support for VSCode. +- [Prettier - Code formatter](https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode) - JavaScript / TypeScript formatter. +- [ESLint](https://marketplace.visualstudio.com/items?itemName=dbaeumer.vscode-eslint) - linter. +- [Jest Runner](https://marketplace.visualstudio.com/items?itemName=firsttris.vscode-jest-runner) - in-editor test runner. +- [Jest Test Explorer](https://marketplace.visualstudio.com/items?itemName=kavod-io.vscode-jest-test-adapter) - adapter to the VSCode testing UI. +- [rust-analyzer](https://marketplace.visualstudio.com/items?itemName=rust-lang.rust-analyzer) - Rust language support for VSCode. diff --git a/node/README.md b/node/README.md index eeaf4e4a60..299499d202 100644 --- a/node/README.md +++ b/node/README.md @@ -14,16 +14,16 @@ The release of Valkey GLIDE was tested on the following platforms: Linux: -- Ubuntu 22.04.1 (x86_64 and aarch64) -- Amazon Linux 2023 (AL2023) (x86_64) +- Ubuntu 22.04.1 (x86_64 and aarch64) +- Amazon Linux 2023 (AL2023) (x86_64) macOS: -- macOS 14.7 (Apple silicon/aarch_64) +- macOS 14.7 (Apple silicon/aarch_64) Alpine: -- node:alpine (default on aarch64 and x86_64) +- node:alpine (default on aarch64 and x86_64) ## NodeJS supported version diff --git a/node/npm/glide/index.ts b/node/npm/glide/index.ts index 2dda300fb3..5778110b1f 100644 --- a/node/npm/glide/index.ts +++ b/node/npm/glide/index.ts @@ -73,7 +73,7 @@ function loadNativeBinding() { let scope = process.env.scope || "@scope"; if (scope == "@scope") { - scope = "@valkey/" + scope = "@valkey/"; } const nativeBinding = require(`${scope}valkey-glide-${nativeStr}`); diff --git a/node/npm/glide/tests/ExportedSymbols.test.ts b/node/npm/glide/tests/ExportedSymbols.test.ts index 948a2afe43..a3bab9468f 100644 --- a/node/npm/glide/tests/ExportedSymbols.test.ts +++ b/node/npm/glide/tests/ExportedSymbols.test.ts @@ -2,7 +2,7 @@ import { it } from "@jest/globals"; import * as f from "fs/promises"; import { describe } from "node:test"; import * as ts from "typescript"; -import * as glideApi from "../"; //ESM convention, +import * as glideApi from "../"; //ESM convention, describe("Exported Symbols test", () => { it("check excluded symbols are not exported", async () => { @@ -28,30 +28,47 @@ describe("Exported Symbols test", () => { internallyExported.sort(); - const skippedListForExports: string[] = ['AdvancedBaseClientConfiguration', - 'ClusterScanOptions', 'GlideMultiJson' + const skippedListForExports: string[] = [ + "AdvancedBaseClientConfiguration", + "ClusterScanOptions", + "GlideMultiJson", ]; const missingSymbols = internallyExported.filter( - (e: string) => (!exportedSymbolsList.includes(e) && !skippedListForExports.includes(e)), + (e: string) => + !exportedSymbolsList.includes(e) && + !skippedListForExports.includes(e), ); - const glideRsKeyWords: string[] = ['ClusterScanCursor', 'Script', 'createLeakedArray', - 'createLeakedAttribute', 'createLeakedBigint', 'createLeakedDouble', 'createLeakedMap', - 'createLeakedString', 'default']; + const glideRsKeyWords: string[] = [ + "ClusterScanCursor", + "Script", + "createLeakedArray", + "createLeakedAttribute", + "createLeakedBigint", + "createLeakedDouble", + "createLeakedMap", + "createLeakedString", + "default", + ]; const doesNotExistExports = exportedSymbolsList.filter( - (e: string) => (!internallyExported.includes(e) && !glideRsKeyWords.includes(e)), + (e: string) => + !internallyExported.includes(e) && !glideRsKeyWords.includes(e), ); if (missingSymbols.length > 0) { - console.log('The following symbols are exported from npm/glide package but missing ' + - 'from the internal node package export. These symbols might be from glide-rs package'); + console.log( + "The following symbols are exported from npm/glide package but missing " + + "from the internal node package export. These symbols might be from glide-rs package", + ); console.log(missingSymbols); } expect(missingSymbols.length).toBe(0); if (doesNotExistExports.length > 0) { - console.log("Symbols that might be missed from the npm/glide package export:") + console.log( + "Symbols that might be missed from the npm/glide package export:", + ); console.log(doesNotExistExports); }