From 66751fba236f365def5ecb85002ae727d3fb5829 Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Wed, 21 May 2025 14:41:41 +0000 Subject: [PATCH 01/14] Set up scripts for correct build, and arrange the files correctly Signed-off-by: Avi Fenesh --- node/package.json | 154 ++++++++++++++++++++-------------- node/rust-client/Cargo.toml | 2 +- node/rust-client/package.json | 55 +++--------- 3 files changed, 104 insertions(+), 107 deletions(-) diff --git a/node/package.json b/node/package.json index d6a09a6b0b..66648e19a7 100644 --- a/node/package.json +++ b/node/package.json @@ -3,61 +3,96 @@ "description": "General Language Independent Driver for the Enterprise (GLIDE) for Valkey", "main": "build-ts/index.js", "module": "build-ts/index.js", - "types": "./build-ts/index.d.ts", + "types": "build-ts/index.d.ts", + "version": "0.0.0", + "exports": { + ".": { + "import": { + "types": "./build-ts/index.d.ts", + "default": "./build-ts/index.js" + }, + "require": { + "types": "./build-ts/index.d.ts", + "default": "./build-ts/index.js" + } + } + }, + "files": [ + "build-ts/**", + "README.md", + "LICENSE" + ], "type": "commonjs", "repository": { "type": "git", "url": "git+https://github.com/valkey-io/valkey-glide.git" }, - "homepage": "https://github.com/valkey-io/valkey-glide#readme", + "homepage": "https://valkey.io/valkey-glide/node/", + "author": "Valkey GLIDE Maintainers", + "license": "Apache-2.0", "dependencies": { - "glide-rs": "file:rust-client", "long": "5", "protobufjs": "7" }, - "bundleDependencies": [ - "glide-rs" + "keywords": [ + "valkey", + "redis", + "client", + "driver", + "database", + "napi", + "typescript", + "rust", + "cross-platform", + "cluster", + "standalone", + "high-availability", + "performance", + "open-source", + "fault-tolerant", + "distributed", + "scalable", + "resilient", + "valkey-glide" ], "scripts": { - "build": "npm run prereq && npm run build-internal && npm run build-protobuf && npm run build-external", - "build:release": "npm run build-internal:release && npm run build-protobuf && npm run build-external:release", - "build:benchmark": "npm run build-internal:benchmark && npm run build-protobuf && npm run build-external", - "build-internal": "cd rust-client && npm run build", - "build-internal:release": "cd rust-client && npm run build:release", - "build-internal:benchmark": "cd rust-client && npm run build:benchmark", - "build-external": "rm -rf build-ts && tsc && npm run copy-protobuf-type", - "build-external:release": "rm -rf build-ts && tsc --stripInternal && npm run copy-protobuf-type", - "build-protobuf": "npm run compile-protobuf-files && npm run fix-protobuf-file", - "copy-protobuf-type": "cp src/ProtobufMessage.d.ts build-ts/src/ProtobufMessage.d.ts", - "compile-protobuf-files": "cd src && pbjs -t static-module -o ProtobufMessage.js ../../glide-core/src/protobuf/*.proto && pbts -o ProtobufMessage.d.ts ProtobufMessage.js", - "clean": "rm -rf build-ts rust-client/target docs glide-logs rust-client/glide-rs.*.node rust-client/index.* src/ProtobufMessage.*", - "docs": "npm run build && ./docs/build-docs", - "fix-protobuf-file": "replace 'this\\.encode\\(message, writer\\)\\.ldelim' 'this.encode(message, writer && writer.len ? writer.fork() : writer).ldelim' src/ProtobufMessage.js", + "clean:build": "rm -rf build-ts dist/*/target rust-client/target rust-client/valkey-glide.*.node src/valkey-glide.*.node build/*.tsbuildinfo", + "clean": "rm -rf build-ts dist/*/target node_modules rust-client/node_modules */node_modules rust-client/target glide-logs rust-client/valkey-glide.*.node src/valkey-glide.*.node rust-client/index.* src/ProtobufMessage.* *.tsbuildinfo test-report*.html yarn.lock package-lock.json rust-client/package-lock.json rust-client/Cargo.lock", + "prebuild": "npm run clean:build", + "build": "BUILD_MODE=dev npm run build:flow", + "build:release": "BUILD_MODE=release npm run build:flow", + "build:benchmark": "BUILD_MODE=benchmark npm run build:flow", + "build:flow": "npm run prereq && npm run build-protobuf && npm run build:rust-client && npm run build:ts", + "build:rust-client": "cd rust-client && npm run build:${BUILD_MODE:-dev}", + "build:ts": "tsc", + "build:ts:release": "tsc --stripInternal", + "build-protobuf": "mkdir -p build-ts && pbjs -t static-module -w commonjs --no-verify --no-convert -o build-ts/ProtobufMessage.js ../glide-core/src/protobuf/*.proto && pbts -o build-ts/ProtobufMessage.d.ts build-ts/ProtobufMessage.js && replace 'this\\.encode\\(message, writer\\)\\.ldelim' 'this.encode(message, writer && writer.len ? writer.fork() : writer).ldelim' build-ts/ProtobufMessage.js", "test": "npm run build-test-utils && jest --verbose --testPathIgnorePatterns='ServerModules'", - "test-local": "npm run build-test-utils && jest --verbose --testPathIgnorePatterns='Validation of Exported Symbols' --testPathIgnorePatterns='ServerModules'", - "test-dbg": "npm run build-test-utils && jest --runInBand", - "test-minimum": "npm run build-test-utils && jest --verbose --runInBand --testNamePattern='^(.(?!(GlideJson|GlideFt|pubsub|kill)))*$'", - "test-modules": "npm run build-test-utils && jest --verbose --runInBand --testNamePattern='(GlideJson|GlideFt)'", + "test:debug": "npm run build-test-utils && jest --runInBand", + "test:minimum": "npm run build-test-utils && jest --verbose --runInBand --testNamePattern='^(.(?!(GlideJson|GlideFt|pubsub|kill)))*$'", + "test:modules": "npm run build-test-utils && jest --verbose --runInBand --testNamePattern='(GlideJson|GlideFt)'", "build-test-utils": "cd ../utils && npm i && npm run build", - "lint:fix": "npm run install-linting && npx eslint -c ../eslint.config.mjs --fix && npm run prettier:format", - "lint": "npm run install-linting && npx eslint -c ../eslint.config.mjs && npm run prettier:check:ci", - "install-linting": "cd ../ & npm install", - "prepack": "npmignore --auto", + "repl": "ts-node --project tsconfig.json", + "lint": "npm run install-linting && eslint -c ../eslint.config.mjs && npm run prettier:check", + "lint:fix": "npm run install-linting && eslint -c ../eslint.config.mjs --fix && npm run prettier:format", + "install-linting": "(cd ../ && npm install)", + "prettier:check": "prettier --check . --ignore-unknown '!**/*.{js,d.ts}'", + "prettier:format": "prettier --write . --ignore-unknown '!**/*.{js,d.ts}'", + "staged": "lint-staged", "prereq": "npm install", - "prettier:check:ci": "npx prettier --check . --ignore-unknown '!**/*.{js,d.ts}'", - "prettier:format": "npx prettier --write . --ignore-unknown '!**/*.{js,d.ts}'" + "artifacts": "napi artifacts", + "prepublishOnly": "cd ../.. && napi prepublish --config npm/glide/package.json -t npm --skip-gh-release" }, "devDependencies": { "@jest/globals": "29", "@types/jest": "29", "@types/minimist": "1", "@types/semver": "7", - "@types/uuid": "10.0.0", - "detect-libc": "2", - "find-free-port": "2.0.0", + "@types/uuid": "10", + "find-free-port": "2", + "lint-staged": "16", "jest": "29", "jest-html-reporter": "4", - "npmignore": "0.3", "protobufjs-cli": "1", "replace": "1", "semver": "7", @@ -68,38 +103,33 @@ "typescript": "5", "uuid": "11" }, - "author": "Valkey GLIDE Maintainers", - "license": "Apache-2.0", - "publishConfig": { - "${registry_scope}registry": "https://registry.npmjs.org/", - "ignore": [ - "src/**", - "tests/", - "rust-client/**", - "!build-ts/**", - ".prettierignore", - "jest.config.js", - "hybrid-node-tests/**", - "docs/", - "DEVELOPER.md", - ".ort.yml", - "tsconfig.json", - "THIRD_PARTY_LICENSES_NODE" + "lint-staged": { + "*.@(js|ts|tsx)": [ + "npx eslint -c ../eslint.config.mjs --fix" + ], + "*.@(js|ts|tsx|yml|yaml|md|json)": [ + "prettier --write" ] }, + "publishConfig": { + "registry": "https://registry.npmjs.org/", + "access": "public" + }, "engines": { "node": ">=16" }, - "//": [ - "The fields below have been commented out and are only necessary for publishing the package." - ], - "///cpu": [ - "${node_arch}" - ], - "///os": [ - "${node_os}" - ], - "///name": "${scope}${pkg_name}", - "///version": "${package_version}", - "//libc": "${libc}" + "napi": { + "name": "valkey-glide", + "triples": { + "defaults": false, + "additional": [ + "x86_64-apple-darwin", + "aarch64-apple-darwin", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-musl" + ] + } + } } diff --git a/node/rust-client/Cargo.toml b/node/rust-client/Cargo.toml index e69d7fefd3..bb72343592 100644 --- a/node/rust-client/Cargo.toml +++ b/node/rust-client/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "glide-rs" +name = "valkey-glide" version = "0.1.0" edition = "2024" license = "Apache-2.0" diff --git a/node/rust-client/package.json b/node/rust-client/package.json index ef71de37fe..5fb585fb81 100644 --- a/node/rust-client/package.json +++ b/node/rust-client/package.json @@ -1,50 +1,21 @@ { - "name": "glide-rs", + "name": "valkey-glide", "version": "0.1.0", - "description": "Valkey client", - "main": "index.js", + "description": "node binary for valkey-glide", "license": "Apache-2.0", - "files": [ - "index.d.ts", - "index.js", - "glide*.*.node" - ], - "napi": { - "name": "glide-rs", - "triples": { - "default": false, - "additional": [ - "x86_64-unknown-linux-musl", - "x86_64-unknown-linux-gnu", - "aarch64-unknown-linux-gnu", - "aarch64-apple-darwin", - "x86_64-apple-darwin", - "aarch64-unknown-linux-musl" - ] - } - }, - "engines": { - "node": ">= 10" - }, - "publishConfig": { - "registry": "https://registry.npmjs.org/", - "access": "public" - }, "scripts": { - "artifacts": "napi artifacts", - "build": "npm install && napi build --features testing_utilities --platform --pipe \"prettier -w\" $npm_config_build_flags", - "build:release": "npm run globals && npm install --omit=dev && npm run globals && napi build --platform --release --strip --pipe \"prettier -w\" $npm_config_build_flags", - "build:benchmark": "npm run globals && npm install && napi build --features testing_utilities --platform --release --pipe \"prettier -w\" $npm_config_build_flags", + "build:dev": "npm install && napi build --features testing_utilities --platform --js ../build-ts/native.js --dts ../build-ts/native.d.ts --js-package-name @valkey/valkey-glide --pipe \"prettier -w\" $npm_config_build_flags && npm run postbuild:copy-node", + "build:benchmark": "npm install && napi build --release --features testing_utilities --platform --js ../build-ts/native.js --dts ../build-ts/native.d.ts --js-package-name @valkey/valkey-glide --pipe \"prettier -w\" $npm_config_build_flags && npm run postbuild:copy-node", + "build:release": "npm install && napi build --release --strip --platform --js ../build-ts/native.js --dts ../build-ts/native.d.ts --js-package-name @valkey/valkey-glide --pipe \"prettier -w\" $npm_config_build_flags && npm run postbuild:copy-node", + "build:release:gnu": "npm install && napi build --release --strip --platform --zig --zig-abi-suffix=2.17 --js ../build-ts/native.js --dts ../build-ts/native.d.ts --js-package-name @valkey/valkey-glide --pipe \"prettier -w\" $npm_config_build_flags", + "postbuild:copy-node": "cp valkey-glide.*.node ../build-ts/", "format": "npm run format:prettier && npm run format:rs", "format:prettier": "prettier . -w", - "format:rs": "cargo fmt", - "prepublishOnly": "napi prepublish -t npm", - "version": "napi version", - "globals": "npm install --global @napi-rs/cli prettier" + "format:rs": "cargo fmt" }, "devDependencies": { "@napi-rs/cli": "2", - "lint-staged": "15", + "lint-staged": "16", "prettier": "3" }, "lint-staged": { @@ -55,11 +26,7 @@ "prettier --write" ] }, - "prettier": { - "printWidth": 120, - "semi": false, - "trailingComma": "all", - "singleQuote": true, - "arrowParens": "always" + "napi": { + "name": "valkey-glide" } } From 841998f4ce6e1691e32246f75db66fe0073d32dd Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Wed, 21 May 2025 14:43:48 +0000 Subject: [PATCH 02/14] arrange npm folder for the new build flow Signed-off-by: Avi Fenesh --- node/npm/darwin-arm64/package.json | 46 +++ node/npm/darwin-x64/package.json | 46 +++ node/npm/glide/.gitignore | 135 ------- node/npm/glide/index.ts | 366 ------------------- node/npm/glide/package.json | 68 ---- node/npm/glide/tests/ExportedSymbols.test.ts | 204 ----------- node/npm/glide/tests/tsconfig.json | 7 - node/npm/glide/tsconfig.json | 30 -- node/npm/linux-arm64-gnu/package.json | 49 +++ node/npm/linux-arm64-musl/package.json | 49 +++ node/npm/linux-x64-gnu/package.json | 49 +++ node/npm/linux-x64-musl/package.json | 49 +++ 12 files changed, 288 insertions(+), 810 deletions(-) create mode 100644 node/npm/darwin-arm64/package.json create mode 100644 node/npm/darwin-x64/package.json delete mode 100644 node/npm/glide/.gitignore delete mode 100644 node/npm/glide/index.ts delete mode 100644 node/npm/glide/package.json delete mode 100644 node/npm/glide/tests/ExportedSymbols.test.ts delete mode 100644 node/npm/glide/tests/tsconfig.json delete mode 100644 node/npm/glide/tsconfig.json create mode 100644 node/npm/linux-arm64-gnu/package.json create mode 100644 node/npm/linux-arm64-musl/package.json create mode 100644 node/npm/linux-x64-gnu/package.json create mode 100644 node/npm/linux-x64-musl/package.json diff --git a/node/npm/darwin-arm64/package.json b/node/npm/darwin-arm64/package.json new file mode 100644 index 0000000000..474f7c8ab4 --- /dev/null +++ b/node/npm/darwin-arm64/package.json @@ -0,0 +1,46 @@ +{ + "name": "@valkey/valkey-glide-darwin-arm64", + "version": "0.0.0", + "repository": { + "type": "git", + "url": "git+https://github.com/valkey-io/valkey-glide.git" + }, + "homepage": "https://valkey.io/valkey-glide/node/", + "author": "Valkey GLIDE Maintainers", + "publishConfig": { + "access": "public" + }, + "description": "Native bindings for valkey-glide on darwin-arm64", + "main": "valkey-glide.darwin-arm64.node", + "files": [ + "valkey-glide.darwin-arm64.node" + ], + "os": [ + "darwin" + ], + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "keywords": [ + "valkey", + "redis", + "client", + "driver", + "database", + "napi", + "typescript", + "rust", + "cross-platform", + "cluster", + "standalone", + "high-availability", + "performance", + "open-source", + "fault-tolerant", + "distributed", + "scalable", + "resilient", + "valkey-glide" + ] +} diff --git a/node/npm/darwin-x64/package.json b/node/npm/darwin-x64/package.json new file mode 100644 index 0000000000..e1ed28f8e2 --- /dev/null +++ b/node/npm/darwin-x64/package.json @@ -0,0 +1,46 @@ +{ + "name": "@valkey/valkey-glide-darwin-x64", + "description": "Native bindings for valkey-glide on darwin-x64", + "version": "0.0.0", + "repository": { + "type": "git", + "url": "git+https://github.com/valkey-io/valkey-glide.git" + }, + "homepage": "https://valkey.io/valkey-glide/node/", + "author": "Valkey GLIDE Maintainers", + "publishConfig": { + "access": "public" + }, + "main": "valkey-glide.darwin-x64.node", + "files": [ + "valkey-glide.darwin-x64.node" + ], + "os": [ + "darwin" + ], + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "keywords": [ + "valkey", + "redis", + "client", + "driver", + "database", + "napi", + "typescript", + "rust", + "cross-platform", + "cluster", + "standalone", + "high-availability", + "performance", + "open-source", + "fault-tolerant", + "distributed", + "scalable", + "resilient", + "valkey-glide" + ] +} diff --git a/node/npm/glide/.gitignore b/node/npm/glide/.gitignore deleted file mode 100644 index 36bedddd60..0000000000 --- a/node/npm/glide/.gitignore +++ /dev/null @@ -1,135 +0,0 @@ -# Generated code -build-ts/** - -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -lerna-debug.log* -.pnpm-debug.log* - -# Diagnostic reports (https://nodejs.org/api/report.html) -report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json - -# Runtime data -pids -*.pid -*.seed -*.pid.lock - -# Directory for instrumented libs generated by jscoverage/JSCover -lib-cov - -# Coverage directory used by tools like istanbul -coverage -*.lcov - -# nyc test coverage -.nyc_output - -# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) -.grunt - -# Bower dependency directory (https://bower.io/) -bower_components - -# node-waf configuration -.lock-wscript - -# Compiled binary addons (https://nodejs.org/api/addons.html) -build/Release - -# Dependency directories -node_modules/ -jspm_packages/ - -# Snowpack dependency directory (https://snowpack.dev/) -web_modules/ - -# TypeScript cache -*.tsbuildinfo - -# Optional npm cache directory -.npm - -# Optional eslint cache -.eslintcache - -# Optional stylelint cache -.stylelintcache - -# Microbundle cache -.rpt2_cache/ -.rts2_cache_cjs/ -.rts2_cache_es/ -.rts2_cache_umd/ - -# Optional REPL history -.node_repl_history - -# Output of 'npm pack' -*.tgz - -# Yarn Integrity file -.yarn-integrity - -# dotenv environment variable files -.env -.env.development.local -.env.test.local -.env.production.local -.env.local - -# parcel-bundler cache (https://parceljs.org/) -.cache -.parcel-cache - -# Next.js build output -.next -out - -# Nuxt.js build / generate output -.nuxt -dist - -# Gatsby files -.cache/ -# Comment in the public line in if your project uses Gatsby and not Next.js -# https://nextjs.org/blog/next-9-1#public-directory-support -# public - -# vuepress build output -.vuepress/dist - -# vuepress v2.x temp and cache directory -.temp -.cache - -# Docusaurus cache and generated files -.docusaurus - -# Serverless directories -.serverless/ - -# FuseBox cache -.fusebox/ - -# DynamoDB Local files -.dynamodb/ - -# TernJS port file -.tern-port - -# Stores VSCode versions used for testing VSCode extensions -.vscode-test - -# yarn v2 -.yarn/cache -.yarn/unplugged -.yarn/build-state.yml -.yarn/install-state.gz -.pnp.* - -package.json.tmpl diff --git a/node/npm/glide/index.ts b/node/npm/glide/index.ts deleted file mode 100644 index 76998967d9..0000000000 --- a/node/npm/glide/index.ts +++ /dev/null @@ -1,366 +0,0 @@ -#!/usr/bin/env node - -/** - * Copyright Valkey GLIDE Project Contributors - SPDX Identifier: Apache-2.0 - */ - -import { MUSL, familySync } from "detect-libc"; -import { arch, platform } from "process"; - -let globalObject = global as unknown; - -/* eslint-disable @typescript-eslint/no-require-imports */ -function loadNativeBinding() { - let nativeStr = process.env.native_binding; - - if (nativeStr == undefined) { - const prefix = familySync() == MUSL ? "-musl" : ""; - nativeStr = `${platform}${prefix}-${arch}`; - - if ( - !["x64", "arm64"].includes(arch) || - !["linux", "darwin"].includes(platform) - ) { - throw new Error( - `Unsupported OS: ${platform}, architecture: ${arch}`, - ); - } - } - - let scope = process.env.scope || "@scope"; - - if (scope == "@scope") { - scope = "@valkey/"; - } - - const nativeBinding = require(`${scope}valkey-glide-${nativeStr}`); - - if (!nativeBinding) { - throw new Error(`Failed to load native binding`); - } - - return nativeBinding; -} - -function initialize() { - const nativeBinding = loadNativeBinding(); - const { - AggregationType, - BaseScanOptions, - ScanOptions, - ZScanOptions, - HScanOptions, - BitEncoding, - BitFieldGet, - BitFieldIncrBy, - BitFieldOffset, - BitFieldOverflow, - BitFieldSet, - BitFieldSubCommands, - BitOffset, - BitOffsetMultiplier, - BitOffsetOptions, - BitOverflowControl, - BitmapIndexType, - BitwiseOperation, - ConditionalChange, - Decoder, - DecoderOption, - GeoAddOptions, - CoordOrigin, - MemberOrigin, - SearchOrigin, - GeoBoxShape, - GeoCircleShape, - GeoSearchShape, - GeoSearchResultOptions, - GeoSearchStoreResultOptions, - SortOrder, - GeoUnit, - GeospatialData, - GlideClient, - GlideClusterClient, - GlideClientConfiguration, - GlideJson, - JsonBatch, - GlideFt, - Field, - TextField, - TagField, - NumericField, - VectorField, - VectorFieldAttributesFlat, - VectorFieldAttributesHnsw, - FtCreateOptions, - FtSearchOptions, - FtInfoReturnType, - FtAggregateOptions, - FtAggregateLimit, - FtAggregateFilter, - FtAggregateGroupBy, - FtAggregateReducer, - FtAggregateSortBy, - FtAggregateSortProperty, - FtAggregateApply, - FtAggregateReturnType, - FtSearchReturnType, - GlideRecord, - GlideString, - JsonGetOptions, - JsonArrPopOptions, - SortedSetDataType, - StreamEntryDataType, - HashDataType, - FunctionListOptions, - FunctionListResponse, - FunctionStatsSingleResponse, - FunctionStatsFullResponse, - FunctionRestorePolicy, - SlotIdTypes, - SlotKeyTypes, - TimeUnit, - RouteByAddress, - RouteOption, - Routes, - RestoreOptions, - SingleNodeRoute, - PeriodicChecksManualInterval, - PeriodicChecks, - Logger, - Limit, - LolwutOptions, - LPosOptions, - ListDirection, - OpenTelemetry, - ExpireOptions, - FlushMode, - InfoOptions, - InsertPosition, - SetOptions, - ZAddOptions, - InfBoundary, - KeyWeight, - Boundary, - ProtocolVersion, - RangeByIndex, - RangeByScore, - RangeByLex, - ReadFrom, - ServerCredentials, - SortOptions, - StreamGroupOptions, - StreamTrimOptions, - StreamAddOptions, - StreamReadGroupOptions, - StreamReadOptions, - StreamClaimOptions, - StreamPendingOptions, - ClosingError, - ConfigurationError, - ExecAbortError, - ValkeyError, - GlideReturnType, - StreamEntries, - ReturnTypeXinfoStream, - RequestError, - TimeoutError, - ConnectionError, - ClusterTransaction, - Transaction, - Batch, - ClusterBatch, - PubSubMsg, - ScoreFilter, - SignedEncoding, - UnsignedEncoding, - UpdateByScore, - createLeakedArray, - createLeakedAttribute, - createLeakedBigint, - createLeakedDouble, - createLeakedMap, - createLeakedString, - Script, - ObjectType, - ClusterScanCursor, - AdvancedGlideClientConfiguration, - AdvancedGlideClusterClientConfiguration, - BaseClientConfiguration, - GlideClusterClientConfiguration, - LevelOptions, - ReturnTypeRecord, - ReturnTypeMap, - ClusterResponse, - ReturnTypeAttribute, - ReturnTypeJson, - UniversalReturnTypeJson, - Score, - ElementAndScore, - BatchOptions, - ClusterBatchOptions, - ClusterBatchRetryStrategy, - } = nativeBinding; - - module.exports = { - AggregationType, - BaseScanOptions, - ScanOptions, - HScanOptions, - ZScanOptions, - BitEncoding, - BitFieldGet, - BitFieldIncrBy, - BitFieldOffset, - BitFieldOverflow, - BitFieldSet, - BitFieldSubCommands, - BitOffset, - BitOffsetMultiplier, - BitOffsetOptions, - BitOverflowControl, - BitmapIndexType, - BitwiseOperation, - ConditionalChange, - Decoder, - DecoderOption, - GeoAddOptions, - GlideFt, - Field, - TextField, - TagField, - NumericField, - VectorField, - VectorFieldAttributesFlat, - VectorFieldAttributesHnsw, - FtCreateOptions, - FtSearchOptions, - FtInfoReturnType, - FtAggregateOptions, - FtAggregateLimit, - FtAggregateFilter, - FtAggregateGroupBy, - FtAggregateReducer, - FtAggregateSortBy, - FtAggregateSortProperty, - FtAggregateApply, - FtAggregateReturnType, - FtSearchReturnType, - GlideRecord, - GlideJson, - JsonBatch, - GlideString, - JsonGetOptions, - JsonArrPopOptions, - SortedSetDataType, - StreamEntryDataType, - HashDataType, - CoordOrigin, - MemberOrigin, - SearchOrigin, - GeoBoxShape, - GeoCircleShape, - GeoSearchShape, - GeoSearchResultOptions, - GeoSearchStoreResultOptions, - SortOrder, - GeoUnit, - GeospatialData, - GlideClient, - GlideClusterClient, - GlideClientConfiguration, - AdvancedGlideClientConfiguration, - AdvancedGlideClusterClientConfiguration, - FunctionListOptions, - FunctionListResponse, - FunctionStatsSingleResponse, - FunctionStatsFullResponse, - FunctionRestorePolicy, - SlotIdTypes, - SlotKeyTypes, - StreamEntries, - TimeUnit, - ReturnTypeXinfoStream, - RouteByAddress, - RouteOption, - Routes, - RestoreOptions, - SingleNodeRoute, - PeriodicChecksManualInterval, - PeriodicChecks, - Logger, - LolwutOptions, - Limit, - LPosOptions, - ListDirection, - OpenTelemetry, - ExpireOptions, - FlushMode, - InfoOptions, - InsertPosition, - SetOptions, - ZAddOptions, - InfBoundary, - KeyWeight, - Boundary, - ProtocolVersion, - RangeByIndex, - RangeByScore, - RangeByLex, - ReadFrom, - ServerCredentials, - SortOptions, - StreamGroupOptions, - StreamTrimOptions, - StreamAddOptions, - StreamClaimOptions, - StreamReadGroupOptions, - StreamReadOptions, - StreamPendingOptions, - ClosingError, - ConfigurationError, - ExecAbortError, - ValkeyError, - GlideReturnType, - RequestError, - TimeoutError, - ConnectionError, - ClusterTransaction, - Transaction, - Batch, - ClusterBatch, - PubSubMsg, - ScoreFilter, - SignedEncoding, - UnsignedEncoding, - UpdateByScore, - createLeakedArray, - createLeakedAttribute, - createLeakedBigint, - createLeakedDouble, - createLeakedMap, - createLeakedString, - Script, - ObjectType, - ClusterScanCursor, - BaseClientConfiguration, - GlideClusterClientConfiguration, - LevelOptions, - ReturnTypeRecord, - ReturnTypeMap, - ClusterResponse, - ReturnTypeAttribute, - ReturnTypeJson, - UniversalReturnTypeJson, - Score, - ElementAndScore, - BatchOptions, - ClusterBatchOptions, - ClusterBatchRetryStrategy, - }; - - globalObject = Object.assign(global, nativeBinding); -} - -initialize(); - -export default globalObject; diff --git a/node/npm/glide/package.json b/node/npm/glide/package.json deleted file mode 100644 index b4c97ab249..0000000000 --- a/node/npm/glide/package.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "name": "valkey-glide", - "types": "build-ts/index.d.ts", - "version": "${package_version}", - "description": "General Language Independent Driver for the Enterprise (GLIDE) for Valkey", - "main": "build-ts/index.js", - "module": "build-ts/index.js", - "type": "commonjs", - "scripts": { - "lint": "eslint .", - "lint:fix": "eslint . --fix", - "clean": "rm -rf build-ts/", - "copy-declaration-files": "cp ../../build-ts/*.d.ts build-ts/ && cp ../../build-ts/src/*.d.ts build-ts/src/ && cp ../../build-ts/src/server-modules/*.d.ts build-ts/src/server-modules/", - "build": "tsc && mkdir -p build-ts/src && mkdir -p build-ts/src/server-modules && npm run copy-declaration-files" - }, - "files": [ - "/build-ts" - ], - "repository": { - "type": "git", - "url": "git+https://github.com/valkey-io/valkey-glide.git" - }, - "keywords": [ - "valkey", - "valkeyClient", - "client", - "valkey-glide" - ], - "author": "Valkey GLIDE Maintainers", - "license": "Apache-2.0", - "bugs": { - "url": "https://github.com/valkey-io/valkey-glide/issues" - }, - "homepage": "https://github.com/valkey-io/valkey-glide#readme", - "devDependencies": { - "@jest/globals": "^29.7.0", - "@types/node": "^22.13.1", - "@typescript-eslint/eslint-plugin": "^8.24.0", - "@typescript-eslint/parser": "^8.24.0", - "eslint": "^8.31.0", - "typescript": "^5.7.3" - }, - "optionalDependencies": { - "${scope}valkey-glide-darwin-x64": "${package_version}", - "${scope}valkey-glide-darwin-arm64": "${package_version}", - "${scope}valkey-glide-linux-arm64": "${package_version}", - "${scope}valkey-glide-linux-x64": "${package_version}", - "${scope}valkey-glide-linux-musl-arm64": "${package_version}", - "${scope}valkey-glide-linux-musl-x64": "${package_version}" - }, - "eslintConfig": { - "extends": [ - "eslint:recommended", - "plugin:@typescript-eslint/recommended" - ], - "parser": "@typescript-eslint/parser", - "plugins": [ - "@typescript-eslint" - ], - "ignorePatterns": [ - "build-ts/*" - ], - "root": true - }, - "dependencies": { - "detect-libc": "^2.0.3" - } -} diff --git a/node/npm/glide/tests/ExportedSymbols.test.ts b/node/npm/glide/tests/ExportedSymbols.test.ts deleted file mode 100644 index ca89e44892..0000000000 --- a/node/npm/glide/tests/ExportedSymbols.test.ts +++ /dev/null @@ -1,204 +0,0 @@ -/** - * Copyright Valkey GLIDE Project Contributors - SPDX Identifier: Apache-2.0 - */ - -import { describe, it } from "@jest/globals"; -import * as f from "fs/promises"; -import * as ts from "typescript"; -import * as glideApi from "../"; //ESM convention, - -const skippedListForExports: string[] = [ - "AdvancedBaseClientConfiguration", - "ClusterScanOptions", - "GlideMultiJson", - "BaseClient", - "BaseBatch", - "BaseBatchOptions", - "convertFieldsAndValuesToHashDataType", - "parseInfoResponse", - "command_request", - "connection_request", - "response", - "WritePromiseOptions", -]; - -const glideRsKeyWords: string[] = [ - "ClusterScanCursor", - "Script", - "createLeakedArray", - "createLeakedAttribute", - "createLeakedBigint", - "createLeakedDouble", - "createLeakedMap", - "createLeakedString", - "default", -]; - -const skipFolders = [ - "commonjs-test", - "glide-logs", - "hybrid-node-tests", - "node_modules", - "npm", - ".cargo", - "target", - "tests", -]; - -describe("Validation of Exported Symbols", () => { - it("check excluded symbols are not exported", async () => { - // Check exported symbols for valkey glide package - const exportedSymbolsList = Object.keys(glideApi).sort(); // exportedList from the npm/glide package. - - const implBuildFolder = "./build-ts/"; - const filesWithNodeCode = await getFiles(implBuildFolder); - - const internallyExported: string[] = []; - - for (const file of filesWithNodeCode) { - const sourceCode = await f.readFile(file, "utf8"); - const sourceFile = ts.createSourceFile( - file, - sourceCode, - ts.ScriptTarget.Latest, - true, - ); - internallyExported.push(...visitRoot(sourceFile)); - } - - internallyExported.sort(); - - const missingSymbols = internallyExported.filter( - (e: string) => - !exportedSymbolsList.includes(e) && - !skippedListForExports.includes(e), - ); - - const doesNotExistExports = exportedSymbolsList.filter( - (e: string) => - !internallyExported.includes(e) && !glideRsKeyWords.includes(e), - ); - - if (missingSymbols.length > 0) { - console.log( - "The following symbols are exported from npm/glide package but missing from the" + - "internal node package export. These symbols might be from glide-rs package", - ); - console.log(missingSymbols); - } - - expect(missingSymbols.length).toBe(0); - - if (doesNotExistExports.length > 0) { - console.log( - "Symbols that might be missed from the npm/glide package export:", - ); - console.log(doesNotExistExports); - } - - expect(doesNotExistExports.length).toBe(0); - }); -}); - -async function getFiles(folderName: string): Promise { - const files = await f.readdir(folderName, { withFileTypes: true }); - - const filesWithNodeCode = []; - - for (const file of files) { - if (file.isDirectory()) { - if (skipFolders.includes(file.name)) { - continue; - } - - filesWithNodeCode.push( - ...(await getFiles(folderName + file.name + "/")), - ); - } else { - if (!file.name.endsWith(".d.ts")) { - continue; - } - - filesWithNodeCode.push(folderName + file.name); - } - } - - return filesWithNodeCode; -} - -function visitRoot(root: ts.Node) { - // (Root Level)->(Level 1) - const children: readonly ts.Node[] = root.getChildren(); - const resultList: string[] = []; - - // (Root Level) -> (Level 1) -> Level 2. This is the level in the AST where all the exported symbols in a file are present. - for (const node of children) { - const nodeList: string[] = node - .getChildren() - .map((c) => visit(c)) - .filter((c) => c !== undefined); - - if (nodeList.length > 0) { - resultList.push(...nodeList); - } - } - - return resultList; -} - -function visit(node: ts.Node) { - let name: string | undefined = ""; - - // List of exported symbols we want to ignore. - switch (node.kind) { - case ts.SyntaxKind.FirstStatement: - case ts.SyntaxKind.ExportDeclaration: - case ts.SyntaxKind.ExportAssignment: - case ts.SyntaxKind.ImportDeclaration: - return; - } - - // list exported symbols we want to check for, like, InterfaceDeclaration, FunctionDeclaration, etc. - if (ts.isFunctionDeclaration(node)) { - name = node.name?.text; - } else if (ts.isVariableStatement(node)) { - name = ""; - } else if (ts.isInterfaceDeclaration(node)) { - name = node.name?.text; - } else if (ts.isClassDeclaration(node)) { - name = node.name?.text; - } else if (ts.isTypeAliasDeclaration(node)) { - name = node.name?.text; - } else if (ts.isEnumDeclaration(node)) { - name = node.name?.text; - } else if (ts.isModuleDeclaration(node)) { - name = node.name?.text; - } - - const children = node.getChildren(); - const isInternal = children - .find((c) => ts.SyntaxKind[c.kind] == "JSDocComment") - ?.getText() - .includes("@internal"); - const isExported = children - .find((c) => ts.SyntaxKind[c.kind] == "SyntaxList") - ?.getChildren() - .find((c) => ts.SyntaxKind[c.kind] == "ExportKeyword"); - - if (isExported && !isInternal) { - // Not internal symbol exported for external use. - return name; - } - - if (isExported && isInternal) { - // marked correctly... no-op. Exported for internal use in the code. - } - - if (!isExported && isInternal) { - // no-op - } - - if (!isExported && !isInternal) { - // no-op - } -} diff --git a/node/npm/glide/tests/tsconfig.json b/node/npm/glide/tests/tsconfig.json deleted file mode 100644 index 2a8aaebf40..0000000000 --- a/node/npm/glide/tests/tsconfig.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extends": "../tsconfig.json", - "compilerOptions": { - "rootDir": "../" - }, - "include": ["./*.test.ts"] -} diff --git a/node/npm/glide/tsconfig.json b/node/npm/glide/tsconfig.json deleted file mode 100644 index 14c61aaf26..0000000000 --- a/node/npm/glide/tsconfig.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "compilerOptions": { - "target": "ESNext", - "module": "commonjs", - "allowJs": true, - "esModuleInterop": true, - "baseUrl": "./", - "forceConsistentCasingInFileNames": true, - "strict": true, - "skipLibCheck": true, - "moduleResolution": "node" /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */, - "allowSyntheticDefaultImports": true, - "declaration": true, - "inlineSourceMap": false, - "lib": ["esnext"], - "listEmittedFiles": false, - "listFiles": false, - "noFallthroughCasesInSwitch": true, - "pretty": true, - "resolveJsonModule": true, - "traceResolution": false, - "types": ["node", "jest"], - "outDir": "./build-ts" /* Specify an output folder for all emitted files */ - }, - "include": ["./*.ts", "src/*.ts", "src/*.js"], - "exclude": [ - "node_modules", - "build-ts" - ] /* Prevents TypeScript from compiling files in node_modules and build-ts */ -} diff --git a/node/npm/linux-arm64-gnu/package.json b/node/npm/linux-arm64-gnu/package.json new file mode 100644 index 0000000000..e3b2faf40f --- /dev/null +++ b/node/npm/linux-arm64-gnu/package.json @@ -0,0 +1,49 @@ +{ + "name": "@valkey/valkey-glide-linux-arm64-gnu", + "description": "Native bindings for valkey-glide on linux-arm64-gnu", + "version": "0.0.0", + "repository": { + "type": "git", + "url": "git+https://github.com/valkey-io/valkey-glide.git" + }, + "homepage": "https://valkey.io/valkey-glide/node/", + "author": "Valkey GLIDE Maintainers", + "publishConfig": { + "access": "public" + }, + "main": "valkey-glide.linux-arm64-gnu.node", + "files": [ + "valkey-glide.linux-arm64-gnu.node" + ], + "os": [ + "linux" + ], + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "keywords": [ + "valkey", + "redis", + "client", + "driver", + "database", + "napi", + "typescript", + "rust", + "cross-platform", + "cluster", + "standalone", + "high-availability", + "performance", + "open-source", + "fault-tolerant", + "distributed", + "scalable", + "resilient", + "valkey-glide" + ] +} diff --git a/node/npm/linux-arm64-musl/package.json b/node/npm/linux-arm64-musl/package.json new file mode 100644 index 0000000000..a5623d3c4e --- /dev/null +++ b/node/npm/linux-arm64-musl/package.json @@ -0,0 +1,49 @@ +{ + "name": "@valkey/valkey-glide-linux-arm64-musl", + "description": "Native bindings for valkey-glide on linux-arm64-musl", + "version": "0.0.0", + "repository": { + "type": "git", + "url": "git+https://github.com/valkey-io/valkey-glide.git" + }, + "homepage": "https://valkey.io/valkey-glide/node/", + "author": "Valkey GLIDE Maintainers", + "publishConfig": { + "access": "public" + }, + "main": "valkey-glide.linux-arm64-musl.node", + "files": [ + "valkey-glide.linux-arm64-musl.node" + ], + "os": [ + "linux" + ], + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "keywords": [ + "valkey", + "redis", + "client", + "driver", + "database", + "napi", + "typescript", + "rust", + "cross-platform", + "cluster", + "standalone", + "high-availability", + "performance", + "open-source", + "fault-tolerant", + "distributed", + "scalable", + "resilient", + "valkey-glide" + ] +} diff --git a/node/npm/linux-x64-gnu/package.json b/node/npm/linux-x64-gnu/package.json new file mode 100644 index 0000000000..533010a787 --- /dev/null +++ b/node/npm/linux-x64-gnu/package.json @@ -0,0 +1,49 @@ +{ + "name": "@valkey/valkey-glide-linux-x64-gnu", + "description": "Native bindings for valkey-glide on linux-x64-gnu", + "version": "0.0.0", + "repository": { + "type": "git", + "url": "git+https://github.com/valkey-io/valkey-glide.git" + }, + "homepage": "https://valkey.io/valkey-glide/node/", + "author": "Valkey GLIDE Maintainers", + "publishConfig": { + "access": "public" + }, + "main": "valkey-glide.linux-x64-gnu.node", + "files": [ + "valkey-glide.linux-x64-gnu.node" + ], + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "keywords": [ + "valkey", + "redis", + "client", + "driver", + "database", + "napi", + "typescript", + "rust", + "cross-platform", + "cluster", + "standalone", + "high-availability", + "performance", + "open-source", + "fault-tolerant", + "distributed", + "scalable", + "resilient", + "valkey-glide" + ] +} diff --git a/node/npm/linux-x64-musl/package.json b/node/npm/linux-x64-musl/package.json new file mode 100644 index 0000000000..7ae6de9dac --- /dev/null +++ b/node/npm/linux-x64-musl/package.json @@ -0,0 +1,49 @@ +{ + "name": "@valkey/valkey-glide-linux-x64-musl", + "description": "Native bindings for valkey-glide on linux-x64-musl", + "version": "0.0.0", + "repository": { + "type": "git", + "url": "git+https://github.com/valkey-io/valkey-glide.git" + }, + "homepage": "https://valkey.io/valkey-glide/node/", + "author": "Valkey GLIDE Maintainers", + "publishConfig": { + "access": "public" + }, + "main": "valkey-glide.linux-x64-musl.node", + "files": [ + "valkey-glide.linux-x64-musl.node" + ], + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "keywords": [ + "valkey", + "redis", + "client", + "driver", + "database", + "napi", + "typescript", + "rust", + "cross-platform", + "cluster", + "standalone", + "high-availability", + "performance", + "open-source", + "fault-tolerant", + "distributed", + "scalable", + "resilient", + "valkey-glide" + ] +} From ec7161513defa046cf176da8ca8aba3f1d789f9a Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Wed, 21 May 2025 14:45:40 +0000 Subject: [PATCH 03/14] rearrange code to fit new changes, and fix types Signed-off-by: Avi Fenesh --- node/index.ts | 16 ------------ node/src/BaseClient.ts | 33 ++++++++++++------------ node/src/Batch.ts | 2 +- node/src/Commands.ts | 8 ++++-- node/src/GlideClient.ts | 2 +- node/src/GlideClusterClient.ts | 12 +++++---- node/src/Logger.ts | 2 +- node/src/index.ts | 38 +++++++++++++++++++++++++++ node/tsconfig.json | 47 +++++++++++++++------------------- 9 files changed, 90 insertions(+), 70 deletions(-) delete mode 100644 node/index.ts create mode 100644 node/src/index.ts diff --git a/node/index.ts b/node/index.ts deleted file mode 100644 index a8af7a0588..0000000000 --- a/node/index.ts +++ /dev/null @@ -1,16 +0,0 @@ -/** - * Copyright Valkey GLIDE Project Contributors - SPDX Identifier: Apache-2.0 - */ - -export { ClusterScanCursor, OpenTelemetryConfig, Script } from "glide-rs"; -export * from "./src/BaseClient"; -export * from "./src/Batch"; -export * from "./src/Commands"; -export * from "./src/Errors"; -export * from "./src/GlideClient"; -export * from "./src/GlideClusterClient"; -export * from "./src/Logger"; -export * from "./src/OpenTelemetry"; -export * from "./src/server-modules/GlideFt"; -export * from "./src/server-modules/GlideFtOptions"; -export * from "./src/server-modules/GlideJson"; diff --git a/node/src/BaseClient.ts b/node/src/BaseClient.ts index bc2117838e..6dc44316c5 100644 --- a/node/src/BaseClient.ts +++ b/node/src/BaseClient.ts @@ -1,26 +1,29 @@ /** * Copyright Valkey GLIDE Project Contributors - SPDX Identifier: Apache-2.0 */ +import Long from "long"; +import * as net from "net"; +import { + Buffer, + BufferWriter, + Long as ProtoLong, + Reader, + Writer, +} from "protobufjs/minimal"; import { DEFAULT_CONNECTION_TIMEOUT_IN_MILLISECONDS, DEFAULT_INFLIGHT_REQUESTS_LIMIT, DEFAULT_REQUEST_TIMEOUT_IN_MILLISECONDS, Script, StartSocketConnection, - createLeakedOtelSpan, - dropOtelSpan, getStatistics, valueFromSplitPointer, -} from "glide-rs"; -import Long from "long"; -import * as net from "net"; +} from "../build-ts/native"; import { - Buffer, - BufferWriter, - Long as ProtoLong, - Reader, - Writer, -} from "protobufjs"; + command_request, + connection_request, + response, +} from "../build-ts/ProtobufMessage"; import { AggregationType, BaseScanOptions, @@ -258,12 +261,8 @@ import { Routes, } from "./GlideClusterClient"; import { Logger } from "./Logger"; +import { createLeakedOtelSpan, dropOtelSpan } from "./native"; import { OpenTelemetry } from "./OpenTelemetry"; -import { - command_request, - connection_request, - response, -} from "./ProtobufMessage"; /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ type PromiseFunction = (value?: any) => void; @@ -777,7 +776,7 @@ export interface PubSubMsg { * @see {@link BatchOptions} */ type BaseOptions = RouteOption & DecoderOption; -export type WritePromiseOptions = +type WritePromiseOptions = | BaseOptions | (BaseOptions & (ClusterBatchOptions | BatchOptions)); diff --git a/node/src/Batch.ts b/node/src/Batch.ts index 9179589657..2999e6b733 100644 --- a/node/src/Batch.ts +++ b/node/src/Batch.ts @@ -272,7 +272,7 @@ import { createZUnion, createZUnionStore, } from "./Commands"; -import { command_request } from "./ProtobufMessage"; +import { command_request } from "../build-ts/ProtobufMessage"; /** * Base class encompassing shared commands for both standalone and cluster mode implementations in a Batch. diff --git a/node/src/Commands.ts b/node/src/Commands.ts index df6862e2b3..f745948780 100644 --- a/node/src/Commands.ts +++ b/node/src/Commands.ts @@ -2,8 +2,11 @@ * Copyright Valkey GLIDE Project Contributors - SPDX Identifier: Apache-2.0 */ -import { createLeakedStringVec, MAX_REQUEST_ARGS_LEN } from "glide-rs"; import Long from "long"; +import { + createLeakedStringVec, + MAX_REQUEST_ARGS_LEN, +} from "../build-ts/native"; import { BaseClient, // eslint-disable-line @typescript-eslint/no-unused-vars @@ -17,11 +20,12 @@ import { Score, SortedSetDataType, } from "./BaseClient"; + +import { command_request } from "../build-ts/ProtobufMessage"; /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ import { GlideClient } from "./GlideClient"; /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ import { GlideClusterClient, SingleNodeRoute } from "./GlideClusterClient"; -import { command_request } from "./ProtobufMessage"; import RequestType = command_request.RequestType; diff --git a/node/src/GlideClient.ts b/node/src/GlideClient.ts index 4387a97584..9338f4251b 100644 --- a/node/src/GlideClient.ts +++ b/node/src/GlideClient.ts @@ -61,7 +61,7 @@ import { LolwutOptions, ScanOptions, } from "./Commands"; -import { connection_request } from "./ProtobufMessage"; +import { connection_request } from "../build-ts/ProtobufMessage"; /* eslint-disable-next-line @typescript-eslint/no-namespace */ export namespace GlideClientConfiguration { diff --git a/node/src/GlideClusterClient.ts b/node/src/GlideClusterClient.ts index d06bf61656..29152585ff 100644 --- a/node/src/GlideClusterClient.ts +++ b/node/src/GlideClusterClient.ts @@ -2,9 +2,13 @@ * Copyright Valkey GLIDE Project Contributors - SPDX Identifier: Apache-2.0 */ -import { ClusterScanCursor, Script } from "glide-rs"; import * as net from "net"; -import { Writer } from "protobufjs"; +import { Writer } from "protobufjs/minimal"; +import { ClusterScanCursor, Script } from "../build-ts/native"; +import { + command_request, + connection_request, +} from "../build-ts/ProtobufMessage"; import { AdvancedBaseClientConfiguration, BaseClient, @@ -64,8 +68,6 @@ import { createTime, createUnWatch, } from "./Commands"; -import { command_request, connection_request } from "./ProtobufMessage"; - /** An extension to command option types with {@link Routes}. */ export interface RouteOption { /** @@ -652,7 +654,7 @@ export class GlideClusterClient extends BaseClient { return new Promise((resolve, reject) => { const callbackIdx = this.getCallbackIndex(); this.promiseCallbackFunctions[callbackIdx] = [ - (resolveAns: [ClusterScanCursor, GlideString[]]) => { + (resolveAns: [typeof cursor, GlideString[]]) => { try { resolve([ new ClusterScanCursor(resolveAns[0].toString()), diff --git a/node/src/Logger.ts b/node/src/Logger.ts index acd09dc2dc..23d2fe0159 100644 --- a/node/src/Logger.ts +++ b/node/src/Logger.ts @@ -2,7 +2,7 @@ * Copyright Valkey GLIDE Project Contributors - SPDX Identifier: Apache-2.0 */ -import { InitInternalLogger, Level, log } from "glide-rs"; +import { InitInternalLogger, Level, log } from "../build-ts/native"; const LEVEL = new Map([ ["error", Level.Error], diff --git a/node/src/index.ts b/node/src/index.ts new file mode 100644 index 0000000000..a6516c89c5 --- /dev/null +++ b/node/src/index.ts @@ -0,0 +1,38 @@ +/** + * Copyright Valkey GLIDE Project Contributors - SPDX Identifier: Apache-2.0 + * + * This entry point re-exports: + * - Native bindings auto-generated by napi-rs (moved to ./native.js) + * - All TypeScript client-side APIs from ./src/ + */ + +import * as Native from "../build-ts/native"; + +export const { + Level, + MAX_REQUEST_ARGS_LEN, + DEFAULT_REQUEST_TIMEOUT_IN_MILLISECONDS, + DEFAULT_CONNECTION_TIMEOUT_IN_MILLISECONDS, + DEFAULT_INFLIGHT_REQUESTS_LIMIT, + AsyncClient, + StartSocketConnection, + log, + InitInternalLogger, + valueFromSplitPointer, + createLeakedStringVec, + Script, + ClusterScanCursor, + getStatistics, +} = Native; + +// Export TypeScript APIs +export * from "./BaseClient.js"; +export * from "./Batch.js"; +export * from "./Commands.js"; +export * from "./Errors.js"; +export * from "./GlideClient.js"; +export * from "./GlideClusterClient.js"; +export * from "./Logger.js"; +export * from "./server-modules/GlideFt.js"; +export * from "./server-modules/GlideFtOptions.js"; +export * from "./server-modules/GlideJson.js"; diff --git a/node/tsconfig.json b/node/tsconfig.json index 50c2bb2fcf..98db74eee6 100644 --- a/node/tsconfig.json +++ b/node/tsconfig.json @@ -1,40 +1,33 @@ { "compilerOptions": { - "target": "ES6" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */, - "module": "CommonJS" /* Specify what module code is generated. */, - "sourceMap": true /* Create source map files for emitted JavaScript files. */, - "declaration": true /* Generate .d.ts files from TypeScript and JavaScript files in your project. */, - "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */, - "strict": true /* Enable all strict type-checking options. */, - "skipLibCheck": true /* Skip type checking all .d.ts files. */, - "allowJs": true /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */, - "allowSyntheticDefaultImports": true /* Allow 'import x from y' when a module doesn't have a default export. */, - "baseUrl": "./" /* Specify the base directory to resolve non-relative module names. */, - "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */, - "inlineSourceMap": false /* Include sourcemap files inside the emitted JavaScript. */, - "listEmittedFiles": false /* List emitted files. */, - "listFiles": false /* Print names of files part of the compilation */, - "moduleResolution": "node" /* Specify how TypeScript looks up a file from a given module specifier. */, - "noFallthroughCasesInSwitch": true /* Enable error reporting for fallthrough cases in switch statements. */, + "target": "ES2021", + "module": "CommonJS", + "lib": ["ES2021"], + "rootDir": "./src", + "outDir": "./build-ts", + "declaration": true, + "sourceMap": true, + "inlineSourceMap": false, + "esModuleInterop": true, + "moduleResolution": "node", + "allowSyntheticDefaultImports": true, + "allowJs": true, + "resolveJsonModule": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "noFallthroughCasesInSwitch": true, "pretty": true, - "resolveJsonModule": true /* Enable importing .json files. */, - "rootDir": "./" /* Specify the root folder within your source files. */, - "traceResolution": false /* true to have TypeScript print information about its resolution process for each processed file */, - "lib": [ - "esnext" - ] /* Specify a set of bundled library declaration files that describe the target runtime environment. */, - "outDir": "./build-ts" /* Specify an output folder for all emitted files.*/ + "skipLibCheck": true }, "ts-node": { "transpileOnly": true, "compilerOptions": { "module": "CommonJS", - "target": "ES2018", + "target": "ES2021", "esModuleInterop": true }, "esm": true }, - "compileOnSave": false, - "include": ["./*.ts", "src/*.ts", "src/*.js", "types/**/*.d.ts"], - "exclude": ["node_modules", "build-ts"] + "include": ["src/**/*", "src/index.ts", "types/**/*.d.ts"], + "exclude": ["node_modules", "build-ts", "tests", "tests/**/*"] } From cc577cb26e3809a82c5a464d28c9ac214cf10728 Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Wed, 21 May 2025 14:46:31 +0000 Subject: [PATCH 04/14] Arrange tests for the changes of structure Signed-off-by: Avi Fenesh --- node/tests/AuthTest.test.ts | 4 +-- node/tests/GlideClient.test.ts | 8 ++--- node/tests/GlideClientInternals.test.ts | 48 ++++++++++++------------- node/tests/GlideClusterClient.test.ts | 4 +-- node/tests/PubSub.test.ts | 4 +-- node/tests/ScanTest.test.ts | 4 +-- node/tests/ServerModules.test.ts | 4 +-- node/tests/SharedTests.ts | 4 +-- node/tests/TestUtilities.ts | 4 +-- node/tests/tsconfig.json | 17 ++++++--- 10 files changed, 55 insertions(+), 46 deletions(-) diff --git a/node/tests/AuthTest.test.ts b/node/tests/AuthTest.test.ts index ad91afc895..0752f1afa2 100644 --- a/node/tests/AuthTest.test.ts +++ b/node/tests/AuthTest.test.ts @@ -10,14 +10,14 @@ import { expect, it, } from "@jest/globals"; +import { ValkeyCluster } from "../../utils/TestUtils"; import { BaseClientConfiguration, GlideClient, GlideClusterClient, ProtocolVersion, RequestError, -} from ".."; -import { ValkeyCluster } from "../../utils/TestUtils"; +} from "../build-ts"; import { flushAndCloseClient, getServerVersion, diff --git a/node/tests/GlideClient.test.ts b/node/tests/GlideClient.test.ts index 0f6a63c40e..0dcbe2e067 100644 --- a/node/tests/GlideClient.test.ts +++ b/node/tests/GlideClient.test.ts @@ -10,7 +10,8 @@ import { expect, it, } from "@jest/globals"; -import { BufferReader, BufferWriter } from "protobufjs"; +import { BufferReader, BufferWriter } from "protobufjs/minimal"; +import { ValkeyCluster } from "../../utils/TestUtils.js"; import { Batch, Decoder, @@ -24,9 +25,8 @@ import { RequestError, Script, convertGlideRecordToRecord, -} from ".."; -import { ValkeyCluster } from "../../utils/TestUtils.js"; -import { command_request } from "../src/ProtobufMessage"; +} from "../build-ts"; +import { command_request } from "../build-ts/ProtobufMessage"; import { runBaseTests } from "./SharedTests"; import { batchTest, diff --git a/node/tests/GlideClientInternals.test.ts b/node/tests/GlideClientInternals.test.ts index d9cd1759c9..dfdb1c1386 100644 --- a/node/tests/GlideClientInternals.test.ts +++ b/node/tests/GlideClientInternals.test.ts @@ -4,20 +4,11 @@ import { describe, expect, it } from "@jest/globals"; import fs from "fs"; -import { - createLeakedArray, - createLeakedAttribute, - createLeakedBigint, - createLeakedDouble, - createLeakedMap, - createLeakedString, - MAX_REQUEST_ARGS_LEN, -} from "glide-rs"; import Long from "long"; import net from "net"; import os from "os"; import path from "path"; -import { Reader } from "protobufjs"; +import { Reader } from "protobufjs/minimal"; import { BaseClientConfiguration, ClosingError, @@ -33,15 +24,24 @@ import { GlideReturnType, InfoOptions, isGlideRecord, + MAX_REQUEST_ARGS_LEN, RequestError, SlotKeyTypes, TimeUnit, -} from ".."; +} from "../build-ts"; +import { + createLeakedArray, + createLeakedAttribute, + createLeakedBigint, + createLeakedDouble, + createLeakedMap, + createLeakedString, +} from "../build-ts/native"; import { command_request, connection_request, response, -} from "../src/ProtobufMessage"; +} from "../build-ts/ProtobufMessage"; import { convertStringArrayToBuffer } from "./TestUtilities"; const { RequestType, CommandRequest } = command_request; @@ -196,21 +196,21 @@ function closeTestResources( socket.end(); } -async function testWithResources( - testFunction: ( +async function testWithResources( + fn: ( connection: GlideClient | GlideClusterClient, socket: net.Socket, - ) => Promise, - connectionOptions?: BaseClientConfiguration, -) { - const { connection, server, socket } = await getConnectionAndSocket( - undefined, - connectionOptions, - ); - - await testFunction(connection, socket); + ) => Promise, +): Promise { + const { connection, socket, server } = await getConnectionAndSocket(); - closeTestResources(connection, server, socket); + try { + return await fn(connection, socket); + } finally { + socket.destroy(); + connection.close(); + server.close(); + } } async function testWithClusterResources( diff --git a/node/tests/GlideClusterClient.test.ts b/node/tests/GlideClusterClient.test.ts index e8bb833f67..e8d80c387a 100644 --- a/node/tests/GlideClusterClient.test.ts +++ b/node/tests/GlideClusterClient.test.ts @@ -11,6 +11,7 @@ import { it, } from "@jest/globals"; import { gte } from "semver"; +import { ValkeyCluster } from "../../utils/TestUtils"; import { BitwiseOperation, ClusterBatch, @@ -35,8 +36,7 @@ import { SortOrder, convertGlideRecordToRecord, convertRecordToGlideRecord, -} from ".."; -import { ValkeyCluster } from "../../utils/TestUtils"; +} from "../build-ts"; import { runBaseTests } from "./SharedTests"; import { batchTest, diff --git a/node/tests/PubSub.test.ts b/node/tests/PubSub.test.ts index 7233802a2b..49e0a1ceee 100644 --- a/node/tests/PubSub.test.ts +++ b/node/tests/PubSub.test.ts @@ -10,6 +10,7 @@ import { expect, it, } from "@jest/globals"; +import ValkeyCluster from "../../utils/TestUtils"; import { BaseClientConfiguration, ConfigurationError, @@ -22,8 +23,7 @@ import { ProtocolVersion, PubSubMsg, TimeoutError, -} from ".."; -import ValkeyCluster from "../../utils/TestUtils"; +} from "../build-ts"; import { flushAndCloseClient, getRandomKey, diff --git a/node/tests/ScanTest.test.ts b/node/tests/ScanTest.test.ts index 1f520a0b2e..12b303dd58 100644 --- a/node/tests/ScanTest.test.ts +++ b/node/tests/ScanTest.test.ts @@ -3,6 +3,7 @@ */ import { afterAll, afterEach, beforeAll, describe } from "@jest/globals"; +import { ValkeyCluster } from "../../utils/TestUtils.js"; import { ClusterScanCursor, Decoder, @@ -12,8 +13,7 @@ import { GlideString, ObjectType, ProtocolVersion, -} from ".."; -import { ValkeyCluster } from "../../utils/TestUtils.js"; +} from "../build-ts"; import { flushAndCloseClient, getClientConfigurationOption, diff --git a/node/tests/ServerModules.test.ts b/node/tests/ServerModules.test.ts index 8d777bac59..7f331170c7 100644 --- a/node/tests/ServerModules.test.ts +++ b/node/tests/ServerModules.test.ts @@ -9,6 +9,7 @@ import { expect, it, } from "@jest/globals"; +import { ValkeyCluster } from "../../utils/TestUtils"; import { ClusterBatch, ConditionalChange, @@ -29,8 +30,7 @@ import { RequestError, SortOrder, VectorField, -} from ".."; -import { ValkeyCluster } from "../../utils/TestUtils"; +} from "../build-ts"; import { CreateJsonBatchCommands, flushAndCloseClient, diff --git a/node/tests/SharedTests.ts b/node/tests/SharedTests.ts index 4aba742db6..aea3cd2aa7 100644 --- a/node/tests/SharedTests.ts +++ b/node/tests/SharedTests.ts @@ -8,6 +8,7 @@ // represents a running server instance. See first 2 test cases as examples. import { expect, it } from "@jest/globals"; +import { ValkeyCluster } from "../../utils/TestUtils"; import { BaseClientConfiguration, Batch, @@ -57,8 +58,7 @@ import { convertFieldsAndValuesToHashDataType, convertGlideRecordToRecord, parseInfoResponse, -} from ".."; -import { ValkeyCluster } from "../../utils/TestUtils"; +} from "../build-ts"; import { Client, GetAndSetRandomValue, diff --git a/node/tests/TestUtilities.ts b/node/tests/TestUtilities.ts index 13541fb6c4..1efd63a6ef 100644 --- a/node/tests/TestUtilities.ts +++ b/node/tests/TestUtilities.ts @@ -6,6 +6,7 @@ import { expect } from "@jest/globals"; import { exec } from "child_process"; import { Socket } from "net"; import { promisify } from "util"; +import ValkeyCluster from "../../utils/TestUtils"; import { BaseClient, BaseClientConfiguration, @@ -40,8 +41,7 @@ import { TimeUnit, UnsignedEncoding, convertRecordToGlideRecord, -} from ".."; -import ValkeyCluster from "../../utils/TestUtils"; +} from "../build-ts"; const execAsync = promisify(exec); export function getRandomKey() { diff --git a/node/tests/tsconfig.json b/node/tests/tsconfig.json index 7a9023b980..6d595c0e07 100644 --- a/node/tests/tsconfig.json +++ b/node/tests/tsconfig.json @@ -1,10 +1,19 @@ { - "extends": "../tsconfig.json", "compilerOptions": { + "target": "ESNext", + "module": "CommonJS", "rootDir": "../../", + "baseUrl": "../../", + "outDir": "./__tests_out__", + "noEmit": true, + "declaration": false, + "declarationMap": false, + "sourceMap": false, + "lib": ["ESNext"], + "esModuleInterop": true, + "skipLibCheck": true, "types": ["jest", "node"] }, - "include": ["*.ts", "./*.test.ts", "setup.ts"], - "exclude": ["node_modules", "**/*.spec.ts"], - "isolatedModules": true + "include": ["**/*.ts", "../../utils/TestUtils.ts"], + "exclude": ["node_modules", "../build-ts"] } From d5d6d408833eb2e98cef3ef28bd8790252902339 Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Wed, 21 May 2025 14:47:43 +0000 Subject: [PATCH 05/14] Built the CI CD for the new build Signed-off-by: Avi Fenesh --- .github/json_matrices/build-matrix.json | 9 +- .../workflows/build-node-wrapper/action.yml | 97 -- .../node-create-package-file/action.yml | 120 --- .github/workflows/node.yml | 187 ++-- .github/workflows/npm-cd.yml | 895 +++++++++++------- .github/workflows/ort.yml | 22 +- .../workflows/setup-musl-on-linux/action.yml | 60 -- 7 files changed, 707 insertions(+), 683 deletions(-) delete mode 100644 .github/workflows/build-node-wrapper/action.yml delete mode 100644 .github/workflows/node-create-package-file/action.yml delete mode 100644 .github/workflows/setup-musl-on-linux/action.yml diff --git a/.github/json_matrices/build-matrix.json b/.github/json_matrices/build-matrix.json index c473f1b7f4..e6ce0b0850 100644 --- a/.github/json_matrices/build-matrix.json +++ b/.github/json_matrices/build-matrix.json @@ -2,7 +2,7 @@ { "OS": "ubuntu", "NAMED_OS": "linux", - "RUNNER": "ubuntu-22.04", + "RUNNER": "ubuntu-latest", "ARCH": "x64", "TARGET": "x86_64-unknown-linux-gnu", "PACKAGE_MANAGERS": ["pypi", "npm", "maven", "pkg_go_dev"], @@ -13,6 +13,7 @@ "OS": "ubuntu", "NAMED_OS": "linux", "RUNNER": ["self-hosted", "Linux", "ARM64", "ephemeral"], + "CD_RUNNER": "ubuntu-24.04-arm", "ARCH": "arm64", "TARGET": "aarch64-unknown-linux-gnu", "PACKAGE_MANAGERS": ["pypi", "npm", "maven", "pkg_go_dev"], @@ -21,7 +22,7 @@ { "OS": "macos", "NAMED_OS": "darwin", - "RUNNER": "macos-14", + "RUNNER": "macos-15", "ARCH": "arm64", "TARGET": "aarch64-apple-darwin", "PACKAGE_MANAGERS": ["pypi", "npm", "maven", "pkg_go_dev"], @@ -31,6 +32,7 @@ "OS": "macos", "NAMED_OS": "darwin", "RUNNER": "macos-13", + "CD_RUNNER": "macos-15", "ARCH": "x64", "TARGET": "x86_64-apple-darwin", "PACKAGE_MANAGERS": ["pypi", "npm", "maven", "pkg_go_dev"], @@ -43,10 +45,11 @@ "ARCH": "arm64", "TARGET": "aarch64-unknown-linux-musl", "RUNNER": ["self-hosted", "Linux", "ARM64", "ephemeral"], + "CD_RUNNER": "ubuntu-24.04-arm", "IMAGE": "node:lts-alpine", "CONTAINER_OPTIONS": "--user root --privileged --rm", "PACKAGE_MANAGERS": ["npm"], - "languages": ["node"] + "languages": [] }, { "OS": "ubuntu", diff --git a/.github/workflows/build-node-wrapper/action.yml b/.github/workflows/build-node-wrapper/action.yml deleted file mode 100644 index c8afda5cae..0000000000 --- a/.github/workflows/build-node-wrapper/action.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: Build Node wrapper - -inputs: - os: - description: "The current operating system" - required: true - type: string - options: - - amazon-linux - - macos - - ubuntu - named_os: - description: "The name of the current operating system" - required: false - default: "linux" - type: string - options: - - linux - - darwin - arch: - description: "The current architecture" - required: false - default: "x64" - type: string - options: - - x64 - - arm64 - target: - description: "Specified target for rust toolchain, ex. x86_64-apple-darwin" - type: string - required: true - engine-version: - description: "Engine version to install" - required: false - type: string - publish: - description: "Enable building the wrapper in release mode" - required: false - type: boolean - default: "false" - npm_scope: - description: "The NPM scope" - required: false - type: string - default: "@valkey" - github-token: - description: "GITHUB_TOKEN, GitHub App installation access token" - required: true - type: string - -env: - CARGO_TERM_COLOR: always - -runs: - using: "composite" - steps: - - name: Install software dependencies - uses: ./.github/workflows/install-shared-dependencies - with: - os: ${{ inputs.os }} - target: ${{ inputs.target }} - github-token: ${{ inputs.github-token }} - engine-version: ${{ inputs.engine-version }} - - - name: Create package.json file - uses: ./.github/workflows/node-create-package-file - with: - release_version: ${{ env.RELEASE_VERSION }} - os: ${{ inputs.os }} - named_os: ${{ inputs.named_os }} - arch: ${{ inputs.arch }} - npm_scope: ${{ inputs.npm_scope }} - target: ${{ inputs.target }} - - - name: npm install - shell: bash - working-directory: ./node - run: | - rm -rf node_modules && rm -rf package-lock.json && npm install - cd rust-client - rm -rf node_modules && rm -rf package-lock.json && npm install - - name: npm install for /npm/glide package - shell: bash - working-directory: ./node/npm/glide - run: | - rm -rf node_modules && rm -rf package-lock.json && npm install - - name: Build - shell: bash - working-directory: ./node - env: - TARGET: ${{ inputs.target }} - PUBLISH: ${{ inputs.publish }} - run: | - source "$HOME/.cargo/env" - TARGET_FLAG=`if [ "$TARGET" != '' ]; then echo "--target $TARGET"; fi` - BUILD_FLAG=`if [ "$PUBLISH" == 'true' ]; then echo "build:release"; else echo "build"; fi` - npm run $BUILD_FLAG --build-flags="$TARGET_FLAG" diff --git a/.github/workflows/node-create-package-file/action.yml b/.github/workflows/node-create-package-file/action.yml deleted file mode 100644 index 232dcadc73..0000000000 --- a/.github/workflows/node-create-package-file/action.yml +++ /dev/null @@ -1,120 +0,0 @@ -name: Create the package.json main file for the Node wrapper - -inputs: - release_version: - description: "The package release version" - required: true - type: string - os: - description: "The current operating system" - required: true - type: string - options: - - amazon-linux - - macos - - ubuntu - named_os: - description: "The name of the current operating system" - required: false - default: "linux" - type: string - options: - - linux - - darwin - arch: - description: "The current architecture" - required: false - default: "x64" - type: string - options: - - x64 - - arm64 - npm_scope: - description: "The NPM scope" - required: false - type: string - default: "@valkey" - target: - description: "Specified target for rust toolchain, ex. x86_64-apple-darwin" - type: string - required: true - options: - - x86_64-unknown-linux-gnu - - aarch64-unknown-linux-gnu - - x86_64-apple-darwin - - aarch64-apple-darwin - - aarch64-unknown-linux-musl - - x86_64-unknown-linux-musl - -runs: - using: "composite" - steps: - - name: "Setup Environment Variables" - shell: bash - env: - TARGET: ${{ inputs.target }} - NAMED_OS: ${{ inputs.named_os }} - ARCH: ${{ inputs.arch }} - RELEASE_VERSION: ${{ inputs.release_version }} - NPM_SCOPE: ${{ inputs.npm_scope }} - OS: ${{ inputs.os }} - run: | - echo "TARGET=$TARGET" >> $GITHUB_ENV - echo "NAMED_OS=$NAMED_OS" >> $GITHUB_ENV - echo "ARCH=$ARCH" >> $GITHUB_ENV - echo "RELEASE_VERSION=$RELEASE_VERSION" >> $GITHUB_ENV - echo "NPM_SCOPE=$NPM_SCOPE" >> $GITHUB_ENV - echo "OS=$OS" >> $GITHUB_ENV - - - name: Create package.json file - shell: bash - working-directory: ./node - run: | - # echo -musl if inputs.target is musl - export MUSL_FLAG=`if [[ "$TARGET" =~ .*"musl".* ]]; then echo "-musl"; fi` - # set the package name - name="valkey-glide" - # derive the OS and architecture from the inputs - export node_os="$NAMED_OS" - export node_arch="$ARCH" - # set the version - export package_version="$RELEASE_VERSION" - # set the package name - export pkg_name="${name}-${node_os}${MUSL_FLAG}-${node_arch}" - # set the scope - export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi` - # set the registry scope - export registry_scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE:"; fi` - # remove the current name section - if [[ "$OS" =~ .*"macos".* ]]; then - sed '/"name":/d' ./package.json > temp.json && mv temp.json package.json - else - sed -i '/"name":/d' ./package.json - fi - # Remove all `///` occurrences to enable the commented out sections - if [[ "$OS" =~ .*"macos".* ]]; then - sed 's|///||g' package.json > temp.json && mv temp.json package.json - else - sed -i 's|///||g' package.json - fi - # generate package.json from the template - mv package.json package.json.tmpl - envsubst < package.json.tmpl > "package.json" - cat package.json - echo $(ls *json*) - - name: Create package.json file in npm/glide package - shell: bash - working-directory: ./node/npm/glide - run: | - name="valkey-glide" - export node_os="$NAMED_OS" - export node_arch="$ARCH" - export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi` - export MUSL_FLAG=`if [[ "$TARGET" =~ .*"musl".* ]]; then echo "-musl"; fi` - export pkg_name="${name}-${node_os}${MUSL_FLAG}-${node_arch}" - export dev_dependency_name="${scope}${pkg_name}" - # Create package.json and append devDependency - mv package.json package.json.tmpl - envsubst < package.json.tmpl > "package.json" - jq --arg dev_dependency_name "$dev_dependency_name" --arg path "../.." '.devDependencies += {($dev_dependency_name): $path}' package.json > package.tmpl.json && mv package.tmpl.json package.json - cat package.json diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index f87f30f438..cde481845b 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -12,7 +12,6 @@ on: - node/** - utils/cluster_manager.py - .github/workflows/node.yml - - .github/workflows/build-node-wrapper/action.yml - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml @@ -26,7 +25,6 @@ on: - node/** - utils/cluster_manager.py - .github/workflows/node.yml - - .github/workflows/build-node-wrapper/action.yml - .github/workflows/install-shared-dependencies/action.yml - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml @@ -92,10 +90,18 @@ jobs: echo "Job running with the following matrix configuration:" echo "${{ toJson(matrix) }}" + # Install software dependencies + - name: Install shared dependencies + uses: ./.github/workflows/install-shared-dependencies + with: + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: ${{ matrix.engine.version }} + + # Setup Node.js - name: Setup Node uses: actions/setup-node@v4 - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true with: node-version: ${{ matrix.node }} @@ -109,22 +115,43 @@ jobs: ${{ matrix.host.TARGET }}-glide-core ${{ matrix.host.TARGET }} - - name: Build Node wrapper - uses: ./.github/workflows/build-node-wrapper - with: - os: ${{ matrix.host.OS }} - named_os: ${{ matrix.host.NAMED_OS }} - arch: ${{ matrix.host.ARCH }} - target: ${{ matrix.host.TARGET }} - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: ${{ matrix.engine.version }} - - - name: test - run: npm test + # Install npm dependencies + - name: Build working-directory: ./node + run: | + npm install + npm run build:release + + - name: Install Yarn + id: install-yarn + if: ${{ matrix.engine.version == '8.0' && matrix.host.ARCH == 'x64' && matrix.node == '23.x' }} + shell: bash + run: | + curl -o- -L https://yarnpkg.com/install.sh | bash + - name: Add Yarn to PATH + id: add-yarn-to-path + if: ${{ steps.install-yarn.outcome == 'success' }} + run: | + echo "$HOME/.yarn/bin:$HOME/.config/yarn/global/node_modules/.bin" >> $GITHUB_PATH + echo "Yarn PATH configured" - - name: test hybrid node modules - commonjs - if: ${{ matrix.engine.version == '8.0' && matrix.host.OS == 'ubuntu' && matrix.host.RUNNER == 'ubuntu-latest' && matrix.node == '20.x' }} + - name: Test yarn compatibility and correct typescript + if: ${{ steps.add-yarn-to-path.outcome == 'success' }} + working-directory: ./node/pm-and-types-tests + shell: bash + run: | + cd package1 + yarn install + rm -rf node_modules/@valkey/valkey-glide/build-ts/* + cp -r ../../build-ts ./node_modules/@valkey/valkey-glide/ + yarn run build + cd ../package2 + yarn install + yarn run build + + # Additional tests for specific configurations + - name: Test hybrid node modules - commonjs + if: ${{ matrix.engine.version == '8.0' && matrix.node == '23.x' && matrix.host.TARGET == 'x86_64-unknown-linux-gnu' }} run: | npm install npm run test @@ -132,8 +159,8 @@ jobs: env: JEST_HTML_REPORTER_OUTPUT_PATH: test-report-commonjs.html - - name: test hybrid node modules - ecma - if: ${{ matrix.engine.version == '8.0' && matrix.host.OS == 'ubuntu' && matrix.host.RUNNER == 'ubuntu-latest' && matrix.node == '20.x' }} + - name: Test hybrid node modules - ecma + if: ${{ matrix.engine.version == '8.0' && matrix.node == '23.x' && matrix.host.TARGET == 'x86_64-unknown-linux-gnu' }} run: | npm install npm run test @@ -141,17 +168,27 @@ jobs: env: JEST_HTML_REPORTER_OUTPUT_PATH: test-report-ecma.html - - uses: ./.github/workflows/test-benchmark - if: ${{ matrix.engine.version == '8.0' && matrix.host.OS == 'ubuntu' && matrix.host.RUNNER == 'ubuntu-latest' && matrix.node == '20.x' }} + - name: Run benchmarks + uses: ./.github/workflows/test-benchmark + if: ${{ matrix.engine.version == '8.0' && matrix.node == '23.x' && matrix.host.TARGET == 'x86_64-unknown-linux-gnu' }} with: language-flag: -node + - name: Test + shell: bash + run: | + npm run build + npm run test + working-directory: ./node + env: + JEST_HTML_REPORTER_OUTPUT_PATH: test-report-node.html + - name: Upload test reports if: always() continue-on-error: true uses: actions/upload-artifact@v4 with: - name: test-report-node-${{ matrix.node }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.OS }}-${{ matrix.host.ARCH }} + name: node-test-reports-${{ matrix.host.OS }}-${{ matrix.host.ARCH }}-${{ matrix.node }}-${{ github.run_id }} path: | node/test-report*.html utils/clusters/** @@ -200,33 +237,55 @@ jobs: image: ${{ matrix.host.IMAGE }} options: ${{ join(' -q ', matrix.host.CONTAINER_OPTIONS) }} # adding `-q` to bypass empty options steps: - - name: Install git - if: ${{ matrix.host.TARGET == 'x86_64-unknown-linux-musl' }} + - name: Install git and required tools run: | - apk update - apk add --no-cache git tar + if [[ "${{ matrix.host.OS }}" == "amazon-linux" ]]; then + # Basic tools needed for actions to work properly + yum install -y git tar which + yum install -y --allowerasing curl + elif [[ "${{ matrix.host.TARGET }}" == "x86_64-unknown-linux-musl" ]]; then + # Basic tools needed for actions to work properly + apk update + apk add git bash tar curl which build-base python3 + fi + + - name: Create directory structure if needed + run: mkdir -p $GITHUB_WORKSPACE - name: Checkout code - if: ${{ matrix.host.TARGET == 'x86_64-unknown-linux-musl' }} uses: actions/checkout@v4 + with: + fetch-depth: 0 - - name: Setup musl on Linux - if: ${{ matrix.host.TARGET == 'x86_64-unknown-linux-musl' }} - uses: ./.github/workflows/setup-musl-on-linux + - name: Set permissions after checkout + run: chown -R $(id -u):$(id -g) $GITHUB_WORKSPACE + + # Install shared dependencies including Rust and Protocol Buffers + - name: Install shared dependencies + uses: ./.github/workflows/install-shared-dependencies with: - npm-scope: ${{ secrets.NPM_SCOPE }} - npm-auth-token: ${{ secrets.NPM_AUTH_TOKEN }} + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: ${{ matrix.engine.version }} + + # Ensure Rust is in PATH for container environments + - name: Setup Rust PATH + if: ${{ contains(matrix.host.TARGET, 'musl') }} + run: | + echo 'source "$HOME/.cargo/env"' >> ~/.bashrc + source "$HOME/.cargo/env" + echo "PATH=$HOME/.cargo/bin:$PATH" >> $GITHUB_ENV + # Verify Rust is available + which cargo || echo "Cargo still not found in PATH" + echo "Current PATH: $PATH" - name: Setup Node - if: ${{ matrix.host.TARGET == 'x86_64-unknown-linux-musl' }} uses: actions/setup-node@v4 - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true with: node-version: ${{ matrix.node }} - uses: actions/cache@v4 - if: ${{ matrix.host.ARCH != 'arm64' }} # JavaScript Actions in Alpine containers are not supported on arm with: path: | node/rust-client/target @@ -234,21 +293,29 @@ jobs: key: ${{ matrix.host.IMAGE }}-node restore-keys: ${{ matrix.host.IMAGE }} - - name: Build Node wrapper - if: ${{ matrix.host.TARGET == 'x86_64-unknown-linux-musl' }} - uses: ./.github/workflows/build-node-wrapper - with: - os: ${{ matrix.host.OS }} - named_os: ${{ matrix.host.NAMED_OS }} - target: ${{ matrix.host.TARGET }} - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: ${{ matrix.engine.version }} - arch: ${{ matrix.host.ARCH }} - - - name: test - if: ${{ matrix.host.TARGET == 'x86_64-unknown-linux-musl' }} - run: npm test + - name: Build and Test working-directory: ./node + env: + PATH: ${{ env.PATH }}:/root/.cargo/bin:/home/ubuntu/.cargo/bin + run: | + # Verify Rust is available + which cargo || echo "Cargo not found in PATH" + echo "Current PATH: $PATH" + + # Check where cargo is installed + find / -name cargo -type f 2>/dev/null || echo "Could not find cargo executable" + + # Explicitly source cargo env before build + if [ -f "$HOME/.cargo/env" ]; then + source "$HOME/.cargo/env" + echo "Sourced $HOME/.cargo/env" + fi + + # Try to find rustc version + rustc --version || echo "rustc command failed" + + npm run build + npm run test - name: Sanitize IMAGE variable if: ${{ matrix.host.TARGET == 'x86_64-unknown-linux-musl' }} @@ -256,7 +323,6 @@ jobs: run: echo "SANITIZED_IMAGE=${{ matrix.host.IMAGE }}" | sed -r 's/:/-/g' >> $GITHUB_ENV - name: Upload test reports - if: ${{ matrix.host.TARGET == 'x86_64-unknown-linux-musl' }} continue-on-error: true uses: actions/upload-artifact@v4 with: @@ -282,7 +348,7 @@ jobs: - name: Install Node.js uses: actions/setup-node@v4 with: - node-version: 20.x + node-version: latest - uses: actions/cache@v4 with: @@ -294,17 +360,14 @@ jobs: aarch64-unknown-linux-gnu-glide-core aarch64-unknown-linux-gnu - - name: Build Node wrapper - uses: ./.github/workflows/build-node-wrapper - with: - os: ubuntu - named_os: linux - arch: arm64 - target: aarch64-unknown-linux-gnu - github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install and build + working-directory: ./node + run: | + npm install + npm run build - name: test - run: npm run test-modules -- --cluster-endpoints=${{ secrets.MEMDB_MODULES_ENDPOINT }} --tls=true + run: npm run test:modules -- --cluster-endpoints=${{ secrets.MEMDB_MODULES_ENDPOINT }} --tls=true working-directory: ./node - name: Upload test reports diff --git a/.github/workflows/npm-cd.yml b/.github/workflows/npm-cd.yml index d1200d1218..a1d737f2b9 100644 --- a/.github/workflows/npm-cd.yml +++ b/.github/workflows/npm-cd.yml @@ -1,4 +1,4 @@ -# The cross platform build was created based on the [Packaging Rust Applications for the NPM Registry blog](https://blog.orhun.dev/packaging-rust-for-npm/). +# This workflow is responsible for the continuous deployment of the GLIDE project to NPM. name: NPM - Continuous Deployment @@ -6,26 +6,30 @@ on: pull_request: paths: - .github/workflows/npm-cd.yml - - .github/workflows/build-node-wrapper/action.yml - - .github/workflows/start-self-hosted-runner/action.yml - - .github/workflows/install-rust-and-protoc/action.yml - - .github/workflows/install-shared-dependencies/action.yml - - .github/workflows/install-engine/action.yml - .github/json_matrices/** - - .github/workflows/create-test-matrices/action.yml + - node/package.json + - node/npm/** + - node/rust-client/Cargo.toml + - node/rust-client/src/** + - node/src/**/*.ts push: tags: - - "v*.*" + - "v*.*.*" # e.g. v1.2.3, v1.2.3-rc1 workflow_dispatch: inputs: version: description: "The release version of GLIDE, formatted as *.*.* or *.*.*-rc*" required: true publish: - description: "Publish" + description: "Publish to NPM" required: true type: boolean default: false + create_node_tag: + description: "Create git tag with -node suffix" + required: false + type: boolean + default: false concurrency: group: node-cd-${{ github.head_ref || github.ref }}-${{ toJson(inputs) }} @@ -33,403 +37,652 @@ concurrency: permissions: id-token: write + contents: read jobs: - start-self-hosted-runner: - if: github.repository_owner == 'valkey-io' + get-build-parameters: runs-on: ubuntu-latest - environment: AWS_ACTIONS + outputs: + release_version: ${{ steps.get-params.outputs.release_version }} + npm_tag: ${{ steps.get-params.outputs.npm_tag }} + platform_matrix: ${{ steps.load-platform-matrix.outputs.platform_matrix }} + should_publish: ${{ steps.get-params.outputs.should_publish }} + should_create_node_tag: ${{ steps.get-params.outputs.should_create_node_tag }} steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Start self hosted EC2 runner - uses: ./.github/workflows/start-self-hosted-runner - with: - role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} - aws-region: ${{ secrets.AWS_REGION }} - ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }} + - name: Determine event type + id: event-check + shell: bash + env: + GH_EVENT_NAME: ${{ github.event_name }} + GH_REF: ${{ github.ref }} + run: | + echo "Event: $GH_EVENT_NAME Ref: $GH_REF" + echo "event_name=$GH_EVENT_NAME" >> $GITHUB_OUTPUT + echo "ref=$GH_REF" >> $GITHUB_OUTPUT - load-platform-matrix: - runs-on: ubuntu-latest - outputs: - PLATFORM_MATRIX: ${{ steps.load-platform-matrix.outputs.PLATFORM_MATRIX }} - steps: - - name: Checkout - uses: actions/checkout@v4 + - name: Handle pull request event + id: pull-request-params + if: steps.event-check.outputs.event_name == 'pull_request' + shell: bash + run: | + RELEASE="255.255.255" + SHOULD_PUB="false" + SHOULD_TAG="false" + echo "release_version=$RELEASE" >> $GITHUB_OUTPUT + echo "should_publish=$SHOULD_PUB" >> $GITHUB_OUTPUT + echo "should_create_node_tag=$SHOULD_TAG" >> $GITHUB_OUTPUT + echo "Pull request detected - using development version $RELEASE" + + - name: Handle workflow dispatch event + id: workflow-dispatch-params + if: steps.event-check.outputs.event_name == 'workflow_dispatch' + shell: bash + env: + INPUT_VER: ${{ github.event.inputs.version }} + INPUT_PUB: ${{ github.event.inputs.publish }} + INPUT_TAG: ${{ github.event.inputs.create_node_tag }} + run: | + # Validate version exists + if [[ -z "$INPUT_VER" ]]; then + echo "::error::Version input is required for manual workflow runs"; exit 1 + fi + + # Validate version format (semver) + if [[ ! "$INPUT_VER" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-rc[0-9]+)?$ ]]; then + echo "::error::Invalid version format: '$INPUT_VER'. Must be X.Y.Z or X.Y.Z-rcN"; exit 1 + fi - - name: load-platform-matrix + # All good - set variables + RELEASE="$INPUT_VER" + SHOULD_PUB="$INPUT_PUB" + SHOULD_TAG="$INPUT_TAG" + echo "release_version=$RELEASE" >> $GITHUB_OUTPUT + echo "should_publish=$SHOULD_PUB" >> $GITHUB_OUTPUT + echo "should_create_node_tag=$SHOULD_TAG" >> $GITHUB_OUTPUT + echo "Manual workflow run - using version $RELEASE (publish=$SHOULD_PUB, tag=$SHOULD_TAG)" + + - name: Handle tag push event + id: tag-push-params + if: steps.event-check.outputs.event_name == 'push' && startsWith(steps.event-check.outputs.ref, 'refs/tags/v') + shell: bash + run: | + if [[ "${{ steps.event-check.outputs.ref }}" =~ ^refs/tags/v([0-9]+\.[0-9]+\.[0-9]+(-rc[0-9]+)?)$ ]]; then + RELEASE="${BASH_REMATCH[1]}" + SHOULD_PUB="true" + SHOULD_TAG="false" # Auto tag creation only for manual workflow runs + echo "release_version=$RELEASE" >> $GITHUB_OUTPUT + echo "should_publish=$SHOULD_PUB" >> $GITHUB_OUTPUT + echo "should_create_node_tag=$SHOULD_TAG" >> $GITHUB_OUTPUT + echo "Tag push detected - using version $RELEASE from tag" + else + echo "::error::Invalid tag format. Expected vX.Y.Z or vX.Y.Z-rcN"; exit 1 + fi + + - name: Determine npm tag + id: determine-npm-tag + shell: bash + env: + RELEASE: ${{ steps.pull-request-params.outputs.release_version || steps.workflow-dispatch-params.outputs.release_version || steps.tag-push-params.outputs.release_version }} + run: | + # npm-tag logic - simple latest/next determination + if [[ "$RELEASE" == *-rc* ]]; then + # Any release candidate version gets "next" tag + TAG="next" + else + # Regular releases get "latest" tag + TAG="latest" + fi + + echo "Using npm tag: $TAG" + echo "npm_tag=$TAG" >> $GITHUB_OUTPUT + + - name: Consolidate build parameters + id: get-params + shell: bash + env: + RELEASE_VERSION: ${{ steps.pull-request-params.outputs.release_version || steps.workflow-dispatch-params.outputs.release_version || steps.tag-push-params.outputs.release_version }} + NPM_TAG: ${{ steps.determine-npm-tag.outputs.npm_tag }} + SHOULD_PUBLISH: ${{ steps.pull-request-params.outputs.should_publish || steps.workflow-dispatch-params.outputs.should_publish || steps.tag-push-params.outputs.should_publish }} + SHOULD_CREATE_NODE_TAG: ${{ steps.pull-request-params.outputs.should_create_node_tag || steps.workflow-dispatch-params.outputs.should_create_node_tag || steps.tag-push-params.outputs.should_create_node_tag }} + run: | + echo "release_version=$RELEASE_VERSION" >> $GITHUB_OUTPUT + echo "npm_tag=$NPM_TAG" >> $GITHUB_OUTPUT + echo "should_publish=$SHOULD_PUBLISH" >> $GITHUB_OUTPUT + echo "should_create_node_tag=$SHOULD_CREATE_NODE_TAG" >> $GITHUB_OUTPUT + + echo "Final build parameters:" + echo "- Version: $RELEASE_VERSION" + echo "- NPM Tag: $NPM_TAG" + echo "- Publish: $SHOULD_PUBLISH" + echo "- Create Node Tag: $SHOULD_CREATE_NODE_TAG" + + - name: Load Platform Matrix id: load-platform-matrix shell: bash run: | - # Filter entries with npm in PACKAGE_MANAGERS and replace "ephemeral" with "persistent" in RUNNER + # Filter entries with npm in PACKAGE_MANAGERS and use CD_RUNNER if available export PLATFORM_MATRIX=$(jq 'map( select(.PACKAGE_MANAGERS != null and (.PACKAGE_MANAGERS | contains(["npm"]))) - | .RUNNER = ( - if (.RUNNER | type == "array") - then (.RUNNER | map(if . == "ephemeral" then "persistent" else . end)) - else (if .RUNNER == "ephemeral" then "persistent" else .RUNNER end) + | .runner = ( + if (.CD_RUNNER != null) + then .CD_RUNNER + elif (.RUNNER != null and (.RUNNER | type != "array")) then .RUNNER + else "ubuntu-latest" end ) + | .build_type = ( + if (.TARGET | contains("musl")) + then "musl" + elif (.TARGET | contains("gnu")) + then "gnu" + else "mac" + end + ) + | if .RUNNER == "macos13" then .["test-runner"] = "macos13" else . end )' < .github/json_matrices/build-matrix.json | jq -c .) - echo "PLATFORM_MATRIX=${PLATFORM_MATRIX}" >> $GITHUB_OUTPUT + echo "platform_matrix=${PLATFORM_MATRIX}" >> $GITHUB_OUTPUT + echo "Platform matrix loaded: ${PLATFORM_MATRIX}" - publish-binaries: - needs: [start-self-hosted-runner, load-platform-matrix] - if: github.repository_owner == 'valkey-io' - name: Publish packages to NPM - runs-on: ${{ matrix.build.RUNNER }} - container: - image: ${{ matrix.build.IMAGE || '' }} - options: ${{ matrix.build.CONTAINER_OPTIONS || 'none'}} + build-native-modules: + needs: [get-build-parameters] strategy: - fail-fast: false matrix: - build: ${{fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX)}} + include: ${{ fromJson(needs.get-build-parameters.outputs.platform_matrix) }} + fail-fast: false + runs-on: ${{ matrix.runner }} + name: Build for ${{ matrix.TARGET }} (${{ matrix.build_type }}) steps: - - name: Setup self-hosted runner access - if: ${{ contains(matrix.build.RUNNER, 'self-hosted') && matrix.build.TARGET != 'aarch64-unknown-linux-musl' }} - run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide - - # For MUSL on X64 we need to install git since we use the checkout action - - name: Install git for musl - if: ${{ contains(matrix.build.TARGET, 'x86_64-unknown-linux-musl')}} - run: | - apk update - apk add git - - - name: Checkout - if: ${{ matrix.build.TARGET != 'aarch64-unknown-linux-musl' }} + - name: Checkout repository uses: actions/checkout@v4 with: - submodules: "true" fetch-depth: 0 - - name: Setup for musl - if: ${{ contains(matrix.build.TARGET, 'musl')}} - uses: ./.github/workflows/setup-musl-on-linux + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: latest + + - name: Cache cargo registry + uses: actions/cache@v4 + with: + path: | + ~/.cargo + ~/.napi-rs + ./target + key: ${{ matrix.TARGET }}-cargo-cache + restore-keys: | + ${{ matrix.TARGET }}-cargo-cache + cargo-cache + + # You might ask why we don't use the common action for installing Rust, the reason is many targets to install, and caching will be more efficient + - name: Install Rust with the required target + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.TARGET }} + + - name: Install protoc (protobuf) + uses: arduino/setup-protoc@v3 with: - npm-scope: ${{ vars.NPM_SCOPE }} - npm-auth-token: ${{ secrets.NPM_AUTH_TOKEN }} - arch: ${{ matrix.build.ARCH }} + version: "25.1" + repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Set the release version + - name: mkdir for js and d.ts shell: bash run: | - if ${{ env.EVENT_NAME == 'pull_request' }}; then - R_VERSION="255.255.255" - elif ${{ env.EVENT_NAME == 'workflow_dispatch' }}; then - echo "${{env.EVENT_NAME}}" - R_VERSION="${{ env.INPUT_VERSION }}" + # Create a build directory js, and d.ts + mkdir -p ./node/build-ts + + - name: Build Native Modules + working-directory: ./node/rust-client + id: build-action + run: | + npm install -g @napi-rs/cli prettier + + # Set up the env parameters for the build + if [[ "${{ matrix.build_type }}" == "musl" ]]; then + additional_param="--zig" + fi + + # Install needed dependencies + if [[ "${{ matrix.TARGET }}" == *"linux"* ]]; then + sudo apt update + sudo apt install -y snap gcc pkg-config openssl libssl-dev + sudo snap install zig --classic --beta else - R_VERSION=${GITHUB_REF:11} + brew update + brew install tree fi - echo "RELEASE_VERSION=${R_VERSION}" >> $GITHUB_ENV - env: - EVENT_NAME: ${{ github.event_name }} - INPUT_VERSION: ${{ github.event.inputs.version }} - - name: Setup node - if: ${{ !contains(matrix.build.TARGET, 'musl') }} - uses: actions/setup-node@v4 - with: - node-version: "latest" - registry-url: "https://registry.npmjs.org" - architecture: ${{ matrix.build.ARCH }} - scope: "${{ vars.NPM_SCOPE }}" - always-auth: true - token: ${{ secrets.NPM_AUTH_TOKEN }} + # Build the native module - - name: Setup node for publishing - if: ${{ !contains(matrix.build.TARGET, 'musl') }} - working-directory: ./node - run: | - npm config set registry https://registry.npmjs.org/ - npm config set '//registry.npmjs.org/:_authToken' ${{ secrets.NPM_AUTH_TOKEN }} - npm config set scope ${{ vars.NPM_SCOPE }} + # For GNU targets, we use the zig abi param set in the package.json to 2.17 + if [[ "${{ matrix.TARGET }}" == *"gnu"* ]]; then + npm run build:release:gnu - - name: Update package version in config.toml - uses: ./.github/workflows/update-glide-version - with: - folder_path: "${{ github.workspace }}/node/rust-client/.cargo" - named_os: ${{ matrix.build.NAMED_OS }} + # For darwin arm we just build native + elif [[ "${{ matrix.TARGET }}" == "aarch64-apple-darwin" ]]; then + napi build --release --strip --platform --js ../build-ts/native.js --dts ../build-ts/native.d.ts --js-package-name @valkey/valkey-glide --pipe 'prettier -w' $npm_config_build_flags - - name: Build Node wrapper - uses: ./.github/workflows/build-node-wrapper - with: - os: ${{ matrix.build.OS }} - named_os: ${{ matrix.build.NAMED_OS }} - arch: ${{ matrix.build.ARCH }} - target: ${{ matrix.build.TARGET }} - npm_scope: ${{ vars.NPM_SCOPE }} - publish: "true" - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Check if RC and set a distribution tag for the package - shell: bash - run: | - if [[ "${RELEASE_VERSION}" == *"rc"* ]] - then - echo "This is a release candidate: ${RELEASE_VERSION}" - export npm_tag="next" + # For macos x86 we build on mac darwin, since mac allow that easily, for musl we use zig else - echo "This is a stable release: ${RELEASE_VERSION}" - export npm_tag="latest" + napi build --platform --release $additional_param --strip --target "${{ matrix.TARGET }}" --js ../build-ts/native.js --dts ../build-ts/native.d.ts --js-package-name @valkey/valkey-glide --pipe 'prettier -w' $npm_config_build_flags fi - echo "NPM_TAG=${npm_tag}" >> $GITHUB_ENV - - name: Check that the release version dont have typo init - if: ${{ github.event_name != 'pull_request' && contains(env.RELEASE_VERSION, '-') && !contains(env.RELEASE_VERSION, 'rc') }} - run: | - echo "The release version "${RELEASE_VERSION}" contains a typo, please fix it" - echo "The release version should be in the format v{major-version}.{minor-version}.{patch-version}-rc{release-candidate-number} when it a release candidate or v{major-version}.{minor-version}.{patch-version} in a stable release." - exit 1 + # Create directory for this target's node file + mkdir -p ../${{ matrix.TARGET }} - - name: Publish to NPM - if: ${{ github.event_name == 'push' || inputs.publish == true }} - shell: bash - working-directory: ./node - run: | - npm pkg fix - set +e # Disable immediate exit on non-zero exit codes - - # Redirect stderr to stdout, filter out notices and warnings - { npm_publish_err=$(npm publish --tag "${NPM_TAG}" --access public --loglevel=error 2>&1 1>&3- | grep -Ev "notice|ExperimentalWarning|WARN") ;} 3>&1 - publish_exit_code=$? - - # Re-enable immediate exit - set -e - - if [[ $publish_exit_code -eq 0 ]]; then - echo "Package published successfully." - elif echo "$npm_publish_err" | grep -q "You cannot publish over the previously published versions"; then - echo "Skipping publishing, package already published." - elif [[ ! -z "$npm_publish_err" ]]; then - echo "Failed to publish with error: $npm_publish_err" - exit 1 - fi - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} + # Copy just the node file to the target directory + cp -r *.node ../${{ matrix.TARGET }} + + # Report what we've built + echo "Built native module for ${{ matrix.TARGET }}:" + ls -la ../${{ matrix.TARGET }}/ - publish-base-to-npm: - name: Publish the base NPM package - needs: publish-binaries + # Upload the native .node modules as artifacts + - name: Upload Native Modules + uses: actions/upload-artifact@v4 + with: + name: bindings-${{ matrix.TARGET }} + path: ./node/${{ matrix.TARGET }} + retention-days: 1 + if-no-files-found: error + + # We want to use the native.js and native.d.ts files in the base package, but we don't have the glide dir yet so we upload them as artifacts + - name: Upload Module js files + uses: actions/upload-artifact@v4 + if: ${{ matrix.TARGET == 'x86_64-unknown-linux-gnu' }} + with: + name: js-files-${{ github.run_id }} + path: | + ./node/build-ts/native.js + ./node/build-ts/native.d.ts + retention-days: 1 + if-no-files-found: error + + prepare-and-publish: + environment: AWS_ACTIONS + if: github.repository_owner == 'valkey-io' && needs.get-build-parameters.outputs.should_publish == 'true' + needs: [get-build-parameters, build-native-modules] runs-on: ubuntu-latest + outputs: + release_package_artifact_name: "release-packages-${{ github.run_id }}" steps: - - name: Checkout + - name: Checkout repository uses: actions/checkout@v4 with: - submodules: "true" + fetch-depth: 0 + - name: Create Build Directory + shell: bash + run: | + # Create a build directory js, and d.ts + mkdir -p ./node/build-ts - - name: Install node + # Put the native.js and native.d.ts files in the src directory + - name: Download Module js files + uses: actions/download-artifact@v4 + with: + name: js-files-${{ github.run_id }} + path: ./node/build-ts + + # Download all native modules + - name: Download all native modules + uses: actions/download-artifact@v4 + with: + path: ./node/artifacts + + # Setup Node.js for publishing to NPM and caching + - name: Setup Node.js with npm cache uses: actions/setup-node@v4 with: - node-version: "latest" + node-version: latest registry-url: "https://registry.npmjs.org" - scope: "${{ vars.NPM_SCOPE }}" - always-auth: true + scope: "@valkey" + token: ${{ secrets.NPM_AUTH_TOKEN }} + cache: "npm" + cache-dependency-path: node/package.json - - name: Create package.json file + # Install dependencies for the base package + - name: Install dependencies + working-directory: ./node shell: bash - working-directory: ./node/npm/glide run: | - export pkg_name=valkey-glide + # Install only production dependencies to speed up installation + npm install --only=prod - echo "The workflow is: ${{env.EVENT_NAME}}" - if ${{ env.EVENT_NAME == 'workflow_dispatch' }}; then - R_VERSION="${{ env.INPUT_VERSION }}" - else - R_VERSION=${GITHUB_REF:11} - fi - echo "RELEASE_VERSION=${R_VERSION}" >> $GITHUB_ENV - - export package_version=${R_VERSION} - export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi` - mv package.json package.json.tmpl - envsubst < package.json.tmpl > "package.json" - cat package.json - # Fix index.ts based on the scope variable - sed -i "s|@scope/|${scope}|g" index.ts - env: - NPM_SCOPE: ${{ vars.NPM_SCOPE }} - EVENT_NAME: ${{ github.event_name }} - INPUT_VERSION: ${{ github.event.inputs.version }} + # Install only dev dependencies needed for building TypeScript + npm install --no-save typescript protobufjs-cli replace @napi-rs/cli prettier - - name: Build Node wrapper - uses: ./.github/workflows/build-node-wrapper - with: - os: ubuntu - target: "x86_64-unknown-linux-gnu" - github-token: ${{ secrets.GITHUB_TOKEN }} + # Setup npm package directories using NAPI-RS artifacts command + - name: Setup npm package directories + working-directory: ./node + shell: bash + run: | + # Ensure npm directory exists with base glide package + mkdir -p ./npm/glide + + # List downloaded native modules (initial structure after download) + echo "Downloaded native modules (initial structure):" + find ./artifacts -type f -name "*.node" -print | sort + + # Copy LICENSE and README to base package + cp ../LICENSE ./npm/glide/ + cp ./README.md ./npm/glide/ + + # Use NAPI artifacts command to manage native module distribution + echo "Running NAPI artifacts command to copy native modules..." + npm run artifacts + + # Add LICENSE and README to each platform package + echo "Adding LICENSE and README to platform packages..." + for dir in ./npm/*/; do + if [[ "$dir" != "./npm/glide/" ]]; then + platform_dir=$(basename "$dir") + echo "Adding files to $platform_dir package..." + cp ../LICENSE "$dir" + cp ./README.md "$dir" + fi + done + + # Show the final directory structure + echo "Final npm directory structure:" + find ./npm -type f | sort + + # Build TypeScript for base package with optimized settings + - name: Build TypeScript + working-directory: ./node + shell: bash + run: | + # Generate protobuf definitions + npm run build-protobuf - - name: Check if RC and set a distribution tag for the package + # Build TypeScript with optimized settings - preserving documentation + echo "Building TypeScript with optimizations..." + npx tsc --sourceMap false --declarationMap false --stripInternal --pretty --declaration + + # Check the build output + echo "TypeScript build complete. Generated files:" + find ./build-ts -type f | wc -l + du -sh ./build-ts + + # Copy necessary files to npm/glide directory + - name: Prepare base package + working-directory: ./node shell: bash run: | - if [[ ${{ env.RELEASE_VERSION }} == *"rc"* ]] - then - echo "This is a release candidate" - export npm_tag="next" - else - echo "This is a stable release" - export npm_tag="latest" - fi - echo "NPM_TAG=${npm_tag}" >> $GITHUB_ENV + mkdir -p ./npm/glide/build-ts + cp -r ./build-ts/* ./npm/glide/build-ts/ + cp ../LICENSE ./npm/glide/ + cp ./README.md ./npm/glide/ + cp ./package.json ./npm/glide/package.json + + # Set version for base package + - name: Set package versions + shell: bash + working-directory: ./node + env: + NAPI_RS_VERSION_FROM_TAG: ${{ needs.get-build-parameters.outputs.release_version }} + run: | + echo "Setting version in base package using sed..." + cd ./npm/glide + sed -i 's/"version": "0\.0\.0"/"version": "'"${NAPI_RS_VERSION_FROM_TAG}"'"/' package.json + echo "Base package version is now: $(jq -r .version package.json)" + + - name: Install NAPI-RS CLI + shell: bash + run: npm install -g @napi-rs/cli - - name: Publish the base package - if: ${{ github.event_name == 'push' || inputs.publish == true }} + # Use prepublishOnly, which call to napi prepublish -t npm to set up everything properly, versions of native modules reflecting the base package version, and optional dependencies base on triples are added to the base package. + - name: Publish packages + if: ${{ needs.get-build-parameters.outputs.should_publish == 'true' }} shell: bash working-directory: ./node/npm/glide - run: | - # Copy the main README file - cp ../../README.md . - npm install - npm run build - npm publish --access public --tag ${{ env.NPM_TAG }} env: NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} + NPM_TAG: ${{ needs.get-build-parameters.outputs.npm_tag }} + run: | + package_name=$(cat package.json | jq -r .name) + version=$(cat package.json | jq -r .version) + echo "Publishing base package ${package_name}@${version} with tag ${NPM_TAG}" + npm publish --tag ${NPM_TAG} --access public --scope=@valkey + + # Check if the publish was successful + if [ $? -ne 0 ]; then + echo "::error::Base package publish failed"; exit 1 + fi - test-release: - if: ${{ github.event_name == 'push' || inputs.publish == true }} - name: Test the release - needs: [publish-base-to-npm, load-platform-matrix] - runs-on: ${{ matrix.build.RUNNER }} - container: - image: ${{ matrix.build.IMAGE || '' }} - options: ${{ matrix.build.CONTAINER_OPTIONS || 'none'}} - strategy: - fail-fast: false - matrix: - build: ${{fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX)}} + echo "Packages published successfully" + + # Delete artifacts if not published + - name: Delete artifacts + if: always() && ${{ needs.get-build-parameters.outputs.should_publish == 'false' }} + uses: geekyeggo/delete-artifact@v2 + with: + name: | + npm-packages-${{ github.run_id }} + artifacts-* + js-files-${{ github.run_id }} + + create-node-tag: + needs: [get-build-parameters, test-published-release] + # Only create the tag if the publish was successful, tests passed, and the tag creation is requested + if: | + needs.get-build-parameters.outputs.should_publish == 'true' && + needs.get-build-parameters.outputs.should_create_node_tag == 'true' + permissions: + contents: write + runs-on: ubuntu-latest steps: - - name: Setup self-hosted runner access - if: ${{ matrix.build.TARGET == 'aarch64-unknown-linux-gnu' }} - run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} - - name: install redis and git for alpine - if: ${{ contains(matrix.build.TARGET, 'musl') }} + - name: Create and push Node tag + shell: bash + env: + VERSION: ${{ needs.get-build-parameters.outputs.release_version }} run: | - apk update - apk add git redis - node -v + # Create node version tag + NODE_TAG="v${VERSION}-node" + echo "Creating node tag: ${NODE_TAG}" - - name: install Python for ubuntu - if: ${{ contains(matrix.build.TARGET, 'linux-gnu') }} - run: | - sudo apt-get update - sudo apt-get install python3 + # Configure git + git config --global user.name "GitHub Actions" + git config --global user.email "actions@github.com" - - name: Checkout - if: ${{ matrix.build.TARGET != 'aarch64-unknown-linux-musl'}} - uses: actions/checkout@v4 + # Create and push the tag + git tag -a "${NODE_TAG}" -m "Node.js release ${VERSION}" + git push origin "${NODE_TAG}" - - name: Setup for musl - if: ${{ contains(matrix.build.TARGET, 'musl') }} - uses: ./.github/workflows/setup-musl-on-linux - with: - npm-scope: ${{ vars.NPM_SCOPE }} - npm-auth-token: ${{ secrets.NPM_AUTH_TOKEN }} - arch: ${{ matrix.build.ARCH }} + echo "Successfully created and pushed tag: ${NODE_TAG}" - - name: Install engine - if: ${{ !contains(matrix.build.TARGET, 'musl') }} - uses: ./.github/workflows/install-engine - with: - engine-version: "8.0" - target: ${{ matrix.build.target }} + test-published-release: + needs: [get-build-parameters, prepare-and-publish] + # Only run tests if the publish was successful + if: needs.get-build-parameters.outputs.should_publish == 'true' + strategy: + matrix: + include: ${{ fromJson(needs.get-build-parameters.outputs.platform_matrix) }} + fail-fast: true + runs-on: ${{ matrix.test-runner || matrix.runner }} + container: + image: ${{ matrix.IMAGE || ''}} + options: ${{ matrix.CONTAINER_OPTIONS || 'none' }} + name: Test & Cleanup on ${{ matrix.TARGET }} + steps: + - name: Setup musl on Linux + if: ${{ matrix.build_type == 'musl' }} + shell: sh + run: | + apk update + apk add bash git python3 - - name: Setup node - if: ${{ !contains(matrix.build.TARGET, 'musl') }} - uses: actions/setup-node@v4 + - name: Checkout (via action) + if: ${{ !(matrix.TARGET == 'aarch64-unknown-linux-musl') }} + uses: actions/checkout@v4 with: - node-version: "latest" - registry-url: "https://registry.npmjs.org" - architecture: ${{ matrix.build.ARCH }} - scope: "${{ vars.NPM_SCOPE }}" - always-auth: true - token: ${{ secrets.NPM_AUTH_TOKEN }} + fetch-depth: 0 - - name: Install tsc and compile utils + - name: Checkout-action manually repository for musl on ARM64 + if: ${{ matrix.TARGET == 'aarch64-unknown-linux-musl' }} shell: bash - working-directory: ./utils run: | - npm install - npm install -g typescript - npx tsc -p ./tsconfig.json + # First, clone the repository + git clone https://github.com/${{ github.repository }} $GITHUB_WORKSPACE + cd $GITHUB_WORKSPACE + # Now we can run further git commands + git config --global --add safe.directory $GITHUB_WORKSPACE + git fetch origin ${{ github.sha }} + git checkout ${{ github.sha }} + git clean -xdf + git reset --hard - - name: Check if RC and set a distribution tag for the package - shell: bash - run: | - if [[ "${RELEASE_VERSION}" == *"rc"* ]] - then - echo "This is a release candidate" - export npm_tag="next" - else - echo "This is a stable release" - export npm_tag="latest" - fi - echo "NPM_TAG=${npm_tag}" >> $GITHUB_ENV + - name: Setup Node.js + if: ${{ !(matrix.build_type == 'musl') }} + uses: actions/setup-node@v4 + with: + node-version: latest - - name: Run the tests - shell: bash - working-directory: ./utils/release-candidate-testing/node + - name: Install Valkey run: | - npm install - npm install --no-save @valkey/valkey-glide@${{ env.NPM_TAG }} - npm run test - - - name: Deprecating packages on failure - if: ${{ failure() }} - shell: bash - env: - GH_EVENT_NAME: ${{ github.event_name }} - GH_EVENT_INPUT_VERSION: ${{ github.event.inputs.version }} - GH_REF: ${{ github.ref }} - NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} - PLATFORM_MATRIX: ${{ needs.load-platform-matrix.outputs.PLATFORM_MATRIX }} - run: | - # Detect OS and install jq - if [[ "${OSTYPE}" == "darwin"* ]]; then - brew install jq || true - elif command -v apk > /dev/null; then - apk add --no-cache jq + # Install Valkey (preferred) or Redis as fallback + if [[ "${{ matrix.build_type }}" == "gnu" ]]; then + echo "Installing Valkey on GNU/Linux..." + sudo apt update + sudo apt install -y valkey || { + echo "Valkey not found in default repos, trying alternative source..." + curl -fsSL https://packages.redis.io/valkey/setup-valkey | sudo bash - + sudo apt install -y valkey || { + echo "Falling back to Redis..." + sudo apt install -y redis-server + } + } + elif [[ "${{ matrix.build_type }}" == "musl" ]]; then + echo "Installing Valkey on Alpine Linux..." + apk update + apk add --no-cache valkey || apk add --no-cache redis + apk add --no-cache valkey-cli || apk add --no-cache redis-cli + elif [[ "${{ matrix.build_type }}" == "mac" ]]; then + echo "Installing Valkey on macOS..." + brew update + brew install valkey || brew install redis else - sudo apt-get update && sudo apt-get install -y jq + echo "::error::Unsupported target: ${{ matrix.TARGET }} (build_type: ${{ matrix.build_type }})"; exit 1 fi - # Set RELEASE_VERSION correctly using environment variables - if [[ "${GH_EVENT_NAME}" == "workflow_dispatch" ]]; then - RELEASE_VERSION="${GH_EVENT_INPUT_VERSION}" + # Verify installation - prefer valkey-server over redis-server + if command -v valkey-server &> /dev/null; then + echo "✓ Valkey server is available" + elif command -v redis-server &> /dev/null; then + echo "✓ Redis server is available (fallback)" else - RELEASE_VERSION="${GH_REF#refs/tags/v}" + echo "::error::Neither Valkey nor Redis server found"; exit 1 fi - # Validate RELEASE_VERSION - if [[ ! "${RELEASE_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-rc[0-9]+)?$ ]]; then - echo "Invalid release version format: ${RELEASE_VERSION}" - exit 1 - fi - - echo "Release version for Deprecating: ${RELEASE_VERSION}" - - # Deprecating base package - npm deprecate "@valkey/valkey-glide@${RELEASE_VERSION}" "This version has been deprecated" --force || true - - # Process platform matrix - echo "${PLATFORM_MATRIX}" > platform_matrix.json + - name: Run utils/node Tests + working-directory: utils/release-candidate-testing/node + run: | + npm install @valkey/valkey-glide@${{ needs.get-build-parameters.outputs.npm_tag }} --save + npm run build:utils + npm test - while read -r pkg; do - package_name="@valkey/valkey-glide-${pkg}" - echo "Deprecating ${package_name}@${RELEASE_VERSION}" - npm deprecate "${package_name}@${RELEASE_VERSION}" "This version has been deprecated" --force || true - done < <(jq -r '.[] | "\(.NAMED_OS)\(.TARGET | test("musl") | if . then "-musl" else "" end)-\(.ARCH)"' platform_matrix.json) + cleanup-artifacts: + needs: [test-published-release, get-build-parameters] + if: always() && needs.get-build-parameters.outputs.should_publish == 'true' + runs-on: ubuntu-latest + steps: + - name: Delete artifacts + uses: geekyeggo/delete-artifact@v2 + with: + name: | + npm-packages-${{ github.run_id }} + bindings-* + js-files-${{ github.run_id }} + + deprecate-on-failure: + needs: [test-published-release, get-build-parameters] + if: failure() && needs.get-build-parameters.outputs.should_publish == 'true' + runs-on: ubuntu-latest + steps: + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: latest + registry-url: "https://registry.npmjs.org" + token: ${{ secrets.NPM_AUTH_TOKEN }} + scope: "@valkey" - # Reset the repository to make sure we get the clean checkout of the action later in other actions. - # It is not required since in other actions we are cleaning before the action, but it is a good practice to do it here as well. - - name: Reset repository - if: ${{ always() }} && ${{ contains(matrix.build.RUNNER, 'self-hosted') }} - shell: bash + - name: Deprecate Published Packages run: | - git reset --hard - git clean -xdf + echo "Deprecating packages due to failure…" + VERSION="${{ needs.get-build-parameters.outputs.release_version }}" + DEPRECATION_MESSAGE="This version (${VERSION}) was part of a failed release process and should not be used. It may be removed in the future." + + # Function to attempt deprecation with retries + attempt_deprecate() { + local package_spec="$1" + local message="$2" + local retries=5 + local delay=10 # seconds + + echo "Attempting to deprecate ${package_spec}..." + for i in $(seq 1 $retries); do + if npm deprecate "${package_spec}" "${message}"; then + echo "Successfully deprecated ${package_spec}." + return 0 + else + echo "Attempt $i of $retries failed for ${package_spec}. Retrying in ${delay} seconds..." + sleep $delay + fi + done + echo "Note: Could not deprecate ${package_spec} after $retries attempts (it might not have been published or was already deprecated/unpublished)." + return 1 + } + + # Main package + attempt_deprecate "@valkey/valkey-glide@${VERSION}" "${DEPRECATION_MESSAGE}" + + # Platform packages + MATRIX_JSON='${{ needs.get-build-parameters.outputs.platform_matrix }}' + + # Validate JSON before processing + if ! echo "$MATRIX_JSON" | jq -e . > /dev/null 2>&1; then + echo "Warning: Invalid platform matrix JSON, unable to deprecate platform packages." + # Optionally exit 1 here if this should be a hard failure, + # or exit 0 to allow other cleanup steps to run. + exit 0 + fi + + echo "Processing platform packages for deprecation..." + echo "$MATRIX_JSON" | jq -c '.[]' | while read -r target_json_line; do + if [[ -z "$target_json_line" ]]; then + echo "Warning: Skipping empty line from matrix JSON." + continue + fi + + # Extract details from the matrix JSON object for each platform + os=$(echo "$target_json_line" | jq -r '.NAMED_OS // empty') + arch=$(echo "$target_json_line" | jq -r '.ARCH // empty') + build_type=$(echo "$target_json_line" | jq -r '.build_type // empty') # build_type is used to determine -gnu or -musl suffix + + if [[ -n "$os" && -n "$arch" && -n "$build_type" ]]; then + additional_suffix="" + if [[ "$build_type" == "musl" ]]; then + additional_suffix="-musl" + elif [[ "$build_type" == "gnu" ]]; then + additional_suffix="-gnu" + fi + + package_name="@valkey/valkey-glide-${os}-${arch}${additional_suffix}" + attempt_deprecate "${package_name}@${VERSION}" "${DEPRECATION_MESSAGE}" + else + echo "Warning: Incomplete data in matrix object, cannot form package name. Object: $target_json_line" + fi + done + echo "Finished deprecation attempts for platform packages." diff --git a/.github/workflows/ort.yml b/.github/workflows/ort.yml index 85761e8861..9f7e36e0b5 100644 --- a/.github/workflows/ort.yml +++ b/.github/workflows/ort.yml @@ -21,8 +21,8 @@ jobs: github.repository_owner == 'valkey-io' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && !startsWith(github.head_ref, 'ort-diff-for-'))) - strategy: - fail-fast: false + + continue-on-error: true env: PYTHON_ATTRIBUTIONS: "python/THIRD_PARTY_LICENSES_PYTHON" NODE_ATTRIBUTIONS: "node/THIRD_PARTY_LICENSES_NODE" @@ -105,24 +105,6 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v4 - - name: Create package.json file for the Node wrapper - uses: ./.github/workflows/node-create-package-file - with: - release_version: ${{ env.TARGET_COMMIT }} - os: "ubuntu-latest" - - - name: Fix Node base NPM package.json file for ORT - working-directory: ./node/npm/glide - run: | - # Remove the glide-rs dependency to avoid duplication - sed -i '/ "glide-rs":/d' ../../package.json - export pkg_name=valkey-glide-base - export package_version="${{ env.TARGET_COMMIT }}" - export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi` - mv package.json package.json.tmpl - envsubst < package.json.tmpl > "package.json" - cat package.json - - name: Run ORT tools for Node uses: ./.github/workflows/run-ort-tools with: diff --git a/.github/workflows/setup-musl-on-linux/action.yml b/.github/workflows/setup-musl-on-linux/action.yml deleted file mode 100644 index 4f4825cf0f..0000000000 --- a/.github/workflows/setup-musl-on-linux/action.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Setup musl on Linux - -inputs: - npm-scope: - description: "npm scope" - required: true - type: string - npm-auth-token: - description: "npm auth token" - required: true - type: string - arch: - description: "architecture" - required: false - type: string - options: - - "arm64" - - "x64" - default: "arm64" - -runs: - using: "composite" - steps: - - name: Install dependencies - shell: sh - run: | - apk update - apk add bash git sed python3 - - - name: Skip all steps if not on ARM64 - shell: bash - if: ${{ inputs.arch != 'arm64' }} - run: exit 0 - - # Currently "Checkout" action is not supported for musl on ARM64, so the checkout is happening on the runner and - # here we just making sure we getting the clean repo - - name: Clean repository for musl on ARM64 - shell: bash - run: | - git config --global --add safe.directory $GITHUB_WORKSPACE - git fetch origin ${{ github.sha }} - git checkout ${{ github.sha }} - git clean -xdf - git reset --hard - - - name: Set up access for musl on ARM - shell: bash - run: | - chown -R $(whoami):$(whoami) $GITHUB_WORKSPACE - - - name: Setup node - shell: bash - working-directory: ./node - env: - NPM_AUTH_TOKEN: ${{ inputs.npm-auth-token }} - NPM_SCOPE: ${{ inputs.npm-scope }} - run: | - npm config set registry https://registry.npmjs.org/ - npm config set '//registry.npmjs.org/:_authToken' "$NPM_AUTH_TOKEN" - npm config set scope "$NPM_SCOPE" From 86ae8cf9c2cb2df5cc7d7ea6dfb5ef3df59fa8de Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Wed, 21 May 2025 14:48:31 +0000 Subject: [PATCH 06/14] Added test for types, and yarn Signed-off-by: Avi Fenesh --- node/pm-and-types-tests/package1/.yarnrc.yml | 3 ++ node/pm-and-types-tests/package1/package.json | 20 +++++++ node/pm-and-types-tests/package1/src/index.ts | 53 +++++++++++++++++++ node/pm-and-types-tests/package1/src/types.ts | 13 +++++ .../pm-and-types-tests/package1/tsconfig.json | 14 +++++ node/pm-and-types-tests/package2/.yarnrc.yml | 3 ++ node/pm-and-types-tests/package2/package.json | 20 +++++++ node/pm-and-types-tests/package2/src/index.ts | 44 +++++++++++++++ .../pm-and-types-tests/package2/tsconfig.json | 14 +++++ node/pm-and-types-tests/tsconfig.json | 12 +++++ 10 files changed, 196 insertions(+) create mode 100644 node/pm-and-types-tests/package1/.yarnrc.yml create mode 100644 node/pm-and-types-tests/package1/package.json create mode 100644 node/pm-and-types-tests/package1/src/index.ts create mode 100644 node/pm-and-types-tests/package1/src/types.ts create mode 100644 node/pm-and-types-tests/package1/tsconfig.json create mode 100644 node/pm-and-types-tests/package2/.yarnrc.yml create mode 100644 node/pm-and-types-tests/package2/package.json create mode 100644 node/pm-and-types-tests/package2/src/index.ts create mode 100644 node/pm-and-types-tests/package2/tsconfig.json create mode 100644 node/pm-and-types-tests/tsconfig.json diff --git a/node/pm-and-types-tests/package1/.yarnrc.yml b/node/pm-and-types-tests/package1/.yarnrc.yml new file mode 100644 index 0000000000..492613fa1f --- /dev/null +++ b/node/pm-and-types-tests/package1/.yarnrc.yml @@ -0,0 +1,3 @@ +nodeLinker: node-modules + +npmRegistryServer: "https://registry.yarnpkg.com" diff --git a/node/pm-and-types-tests/package1/package.json b/node/pm-and-types-tests/package1/package.json new file mode 100644 index 0000000000..856610b4f3 --- /dev/null +++ b/node/pm-and-types-tests/package1/package.json @@ -0,0 +1,20 @@ +{ + "name": "@test/common", + "version": "1.0.0", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "tsc" + }, + "dependencies": { + "@types/pg": "^8.11.11", + "@valkey/valkey-glide": "next", + "decimal.js": "^10.5.0", + "pg": "^8.14.1", + "winston": "^3.17.0" + }, + "devDependencies": { + "@types/node": "22.15.19", + "typescript": "^5.0.4" + } +} diff --git a/node/pm-and-types-tests/package1/src/index.ts b/node/pm-and-types-tests/package1/src/index.ts new file mode 100644 index 0000000000..9e2e45c61e --- /dev/null +++ b/node/pm-and-types-tests/package1/src/index.ts @@ -0,0 +1,53 @@ +import { GlideClient, GlideClientConfiguration } from "@valkey/valkey-glide"; +import winston from "winston"; + +/** + * A Valkey client wrapper using valkey-glide + */ +export class ValkeyWrapper { + private client: GlideClient; + private logger = winston.createLogger({ + level: "info", + format: winston.format.json(), + transports: [new winston.transports.Console()], + }); + + // make ctor private and accept an already-created client + private constructor(client: GlideClient) { + this.client = client; + } + + // async factory replaces constructor’s await + static async create( + config: GlideClientConfiguration, + ): Promise { + const client = await GlideClient.createClient(config); + return new ValkeyWrapper(client); + } + + async get(key: string): Promise { + try { + return (await this.client.get(key)) as string | null; + } catch (error) { + this.logger.error("Failed to get value from Valkey", error); + throw error; + } + } + + async set(key: string, value: string): Promise { + try { + await this.client.set(key, value); + } catch (error) { + this.logger.error("Failed to set value in Valkey", error); + throw error; + } + } + + disconnect() { + this.client.close(); + this.logger.info("Valkey client disconnected"); + } +} + +// Export other utilities +export * from "./types"; diff --git a/node/pm-and-types-tests/package1/src/types.ts b/node/pm-and-types-tests/package1/src/types.ts new file mode 100644 index 0000000000..2008f5eb03 --- /dev/null +++ b/node/pm-and-types-tests/package1/src/types.ts @@ -0,0 +1,13 @@ +/** + * Types exported by the common package + */ + +export interface DatabaseConnection { + connect(): Promise; + disconnect(): Promise; +} + +export interface CacheOperations { + get(key: string): Promise; + set(key: string, value: T): Promise; +} diff --git a/node/pm-and-types-tests/package1/tsconfig.json b/node/pm-and-types-tests/package1/tsconfig.json new file mode 100644 index 0000000000..88468ca89a --- /dev/null +++ b/node/pm-and-types-tests/package1/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "es2018", + "module": "commonjs", + "declaration": true, + "outDir": "./dist", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/node/pm-and-types-tests/package2/.yarnrc.yml b/node/pm-and-types-tests/package2/.yarnrc.yml new file mode 100644 index 0000000000..492613fa1f --- /dev/null +++ b/node/pm-and-types-tests/package2/.yarnrc.yml @@ -0,0 +1,3 @@ +nodeLinker: node-modules + +npmRegistryServer: "https://registry.yarnpkg.com" diff --git a/node/pm-and-types-tests/package2/package.json b/node/pm-and-types-tests/package2/package.json new file mode 100644 index 0000000000..766923ad6e --- /dev/null +++ b/node/pm-and-types-tests/package2/package.json @@ -0,0 +1,20 @@ +{ + "name": "@test/consumer", + "version": "1.0.0", + "description": "Consumer package that uses @test/common", + "main": "dist/index.js", + "scripts": { + "build": "tsc", + "start": "node dist/index.js", + "dev": "ts-node src/index.ts" + }, + "dependencies": { + "@test/common": "file:../package1", + "winston": "^3.17.0" + }, + "devDependencies": { + "@types/node": "22.15.19", + "ts-node": "^10.9.2", + "typescript": "^5.0.4" + } +} diff --git a/node/pm-and-types-tests/package2/src/index.ts b/node/pm-and-types-tests/package2/src/index.ts new file mode 100644 index 0000000000..0b424e8cc2 --- /dev/null +++ b/node/pm-and-types-tests/package2/src/index.ts @@ -0,0 +1,44 @@ +import { ValkeyWrapper } from "@test/common"; +import winston from "winston"; + +// Set up logger +const logger = winston.createLogger({ + level: "info", + format: winston.format.json(), + transports: [new winston.transports.Console()], +}); + +async function main() { + logger.info("Starting application"); + + // Configure Valkey directly with ClientConfig + try { + // This will use the ValkeyWrapper from @test/common which depends on valkey-glide + const valkeyClient = await ValkeyWrapper.create({ + addresses: [{ host: "localhost", port: 6379 }], + clientName: "test-client", + }); + logger.info("Valkey client connected"); + + // Store a value + await valkeyClient.set("test-key", "Hello from consumer app!"); + logger.info("Value stored in Valkey"); + + // Retrieve the value + const value = await valkeyClient.get("test-key"); + logger.info(`Retrieved value: ${value}`); + + valkeyClient.disconnect(); + logger.info("Valkey client disconnected"); + } catch (error) { + logger.error("Error in application:", error); + process.exit(1); + } +} + +main() + .then(() => logger.info("Application completed successfully")) + .catch((error) => { + logger.error("Unhandled error:", error); + process.exit(1); + }); diff --git a/node/pm-and-types-tests/package2/tsconfig.json b/node/pm-and-types-tests/package2/tsconfig.json new file mode 100644 index 0000000000..ae072a2eec --- /dev/null +++ b/node/pm-and-types-tests/package2/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "es2018", + "module": "commonjs", + "declaration": true, + "outDir": "./dist", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": false, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/node/pm-and-types-tests/tsconfig.json b/node/pm-and-types-tests/tsconfig.json new file mode 100644 index 0000000000..b76200c061 --- /dev/null +++ b/node/pm-and-types-tests/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "es2018", + "module": "commonjs", + "declaration": true, + "outDir": "./dist", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + } +} From 5eeb46f7f9182ef9e85ff4dae0e4e526e9eb9a8c Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Wed, 21 May 2025 14:48:46 +0000 Subject: [PATCH 07/14] Added docs Signed-off-by: Avi Fenesh --- .github/DEVELOPER.md | 2 + .github/NPM-CD.md | 125 +++++++++++++++ node/DEVELOPER.md | 316 ++++++++++++++++++++------------------ node/README.md | 31 ++-- node/rust-client/.ort.yml | 18 +-- 5 files changed, 322 insertions(+), 170 deletions(-) create mode 100644 .github/NPM-CD.md diff --git a/.github/DEVELOPER.md b/.github/DEVELOPER.md index 9b45af17e0..f14a2229e8 100644 --- a/.github/DEVELOPER.md +++ b/.github/DEVELOPER.md @@ -142,3 +142,5 @@ We use dynamic matrices for our CI/CD workflows, which are created using the `cr - `language-version-matrix-output`: The generated language version matrix. This dynamic matrix generation allows for flexible and efficient CI/CD workflows, adapting the test configurations based on the type of change and the specific language being tested. + +> **Note:** For detailed information about the Node.js CD workflow, see the workflow file at `.github/workflows/npm-cd.yml` diff --git a/.github/NPM-CD.md b/.github/NPM-CD.md new file mode 100644 index 0000000000..4d18bee159 --- /dev/null +++ b/.github/NPM-CD.md @@ -0,0 +1,125 @@ +# NPM Continuous Deployment Workflow + +This document describes the Continuous Deployment (CD) workflow for publishing Valkey Glide Node.js packages to npm. + +## Overview + +The NPM CD workflow uses a comprehensive, multi-stage approach to build, package, publish, and test the Valkey Glide Node.js packages. The workflow is defined in a single file (.github/workflows/npm-cd.yml) and includes the following benefits: + +- Comprehensive platform support (Linux with both glibc and musl, macOS) +- Cross-compilation using Zig for Linux targets (with ABI version 2.17 for maximum compatibility) +- Universal binary support for macOS (ARM64 + x86_64) +- Smart version management for different types of releases +- Thorough testing of published packages +- Optimized TypeScript build process preserving documentation +- Automatic deprecation of packages if tests fail after publishing + +## Workflow Structure + +The workflow consists of the following jobs: + +### 1. Parameter Determination (get-build-parameters) + +Determines the appropriate version number, npm tag, and platform matrix based on the workflow trigger: + +- For tags: Extracts version from the tag name (e.g., `v1.2.3` → `1.2.3`) +- For manual triggers: Uses the user-provided version +- For PRs: Uses a placeholder version (255.255.255) +- Sets npm tag based on release type: + - "latest" for stable releases (1.0.0) + - "next" for release candidates (1.0.0-rc1) +- Also determines whether packages should be published based on trigger type + +### 2. Native Module Building (build-native-modules) + +Builds the native bindings for each platform in the matrix: + +- Uses a matrix strategy to build for each supported platform +- Sets up the appropriate build environment (glibc, musl, macOS) +- Installs Rust, Zig, and other required dependencies +- Builds native Node.js modules (.node files) with napi-rs +- Uploads the native modules as artifacts for later assembly +- Uploads JS interface files (native.js/native.d.ts) as artifacts + +### 3. Package Preparation and Publishing (prepare-and-publish) + +Prepares artifacts and publishes packages (when enabled): + +- Downloads native modules and JS interface files +- Uses the NAPI artifacts command to distribute native modules to the correct package folders +- Builds TypeScript source code with optimized settings (--stripInternal --pretty --declaration) +- Sets correct versions in package.json file of the base package. +- Prepares the base package with all TypeScript code +- Copies necessary files (LICENSE, README) to each package +- Publishes the packages with single command, this single command call to prepublishOnly which is a script in package.json, napi prepublish. +- napi prepublish is a script that handles the preparation of the package for publishing, including: + - Aligning the package.json files versions with the base package version. + - Adding the native file to optionalDependencies in package.json. +- Uses npm tag based on version type (latest for stable, next for RC versions) + +### 4. Release Testing (test-published-release) + +Tests the published packages across platforms: + +- Runs a matrix strategy to test on each supported platform +- Installs Valkey (or Redis as fallback) as a prerequisite for testing +- Installs the published package from npm +- Runs utils/node tests to verify functionality +- Reports test results + +### 5. Node Tag Creation (create-node-tag) + +Creates a Node.js-specific git tag when requested: + +- Creates and pushes a tag in the format `vX.Y.Z-node` +- Only runs when explicitly requested via workflow input +- Helps track Node.js-specific releases + +### 6. Cleanup Artifacts (cleanup-artifacts) + +Cleans up artifacts after successful testing: + +- Runs after test-published-release job completes +- Removes all artifacts created during the build process to save storage space +- Deletes native module artifacts, JS files, and package artifacts + +### 7. Failure Handling (deprecate-on-failure) + +Handles failures in the publish or test process: + +- Triggers only if publishing was enabled and previous jobs failed +- Sets up Node.js and NPM with authentication +- Deprecates (not unpublishes, as valkey-glide token don't have such permissions) the base package and all platform packages to warn users +- Uses retry mechanism to handle transient errors with npm registry +- Uses robust JSON parsing to handle potential errors in package inspection +- Marks failed packages with clear deprecation notices explaining the failure + +## TypeScript Build Process + +The workflow includes an optimized TypeScript build process: + +1. Uses npm caching for faster builds +2. Installs only the necessary dependencies for building +3. Compiles TypeScript with the following optimizations: + - `--stripInternal`: Removes @internal marked items from declarations while preserving public documentation + - `--pretty`: Formats error messages for better readability + - `--declaration`: Ensures type declaration files are generated with full JSDoc/TSDoc documentation +4. Reports build statistics for monitoring output size + +## Triggers + +The workflow triggers on: + +1. Pull requests that modify: + - Workflow file itself (.github/workflows/npm-cd.yml) + - JSON matrices (.github/json_matrices/**) + - Package configuration (node/package.json) + - NPM directory structure (node/npm/**) + - Rust client files (node/rust-client/Cargo.toml, node/rust-client/src/**) + - TypeScript sources (node/src/**/*.ts) +2. Pushing tags that match the pattern "v*.*.\*" (e.g., v1.2.3, v1.2.3-rc1) + +3. Manual workflow dispatch with: + - Version input (required) + - Publish option (boolean) + - Create node tag option (boolean) diff --git a/node/DEVELOPER.md b/node/DEVELOPER.md index cba75bf8e2..64664c2aef 100644 --- a/node/DEVELOPER.md +++ b/node/DEVELOPER.md @@ -1,248 +1,260 @@ # Developer Guide -This document describes how to set up your development environment to build and test Valkey GLIDE Node wrapper. +This document describes how to set up your development environment to build and test the **Valkey GLIDE Node wrapper**, part of the polyglot `valkey-glide` project. -### Development Overview +## Project Structure -The GLIDE Node wrapper consists of both TypeScript and Rust code. Rust bindings for Node.js are implemented using [napi-rs](https://github.com/napi-rs/napi-rs). The Node and Rust components communicate using the [protobuf](https://github.com/protocolbuffers/protobuf) protocol. +- `valkey-glide/`: The polyglot root directory. +- `valkey-glide/node/`: Contains the Node.js binding (TypeScript + Rust napi-rs). + - `src/`: TypeScript source code with modular file organization. + - `index.ts`: Main entry point that re-exports all APIs. + - `*.ts`: TypeScript files for client classes, commands, batching, etc.. + - `server-modules/`: Server module implementations (e.g., JSON, FT). + - `native.js`, `native.d.ts`: Auto-generated bindings by napi-rs. + - `build-ts/`: Compiled TypeScript output. + - `tests/`: Jest tests. + - `rust-client/`: napi-rs Rust crate (binds to `glide-core`). + - `npm/`: Platform-specific packages for CD. + - Platform-specific directories (e.g., `linux-arm64-gnu/`, `darwin-arm64/`) containing package.json and native binaries. -### Build from source +## Development Overview -#### Prerequisites +The Node.js wrapper consists of: -Software Dependencies +- TypeScript: User-facing API organized in logical modules. +- Rust: Native module built with [`napi-rs`](https://github.com/napi-rs/napi-rs). +- Communication: Protocol Buffers for efficient data exchange. -##### **Note:** Nodejs Supported Version +### Build scripts overview -If your Nodejs version is below the supported version specified in the client's [documentation](https://github.com/valkey-io/valkey-glide/blob/main/node/README.md#nodejs-supported-version), you can upgrade it using [NVM](https://github.com/nvm-sh/nvm?tab=readme-ov-file#install--update-script). +The build process consists of multiple steps: +1. `clean:build` - Removes previous build artifacts +2. `build-protobuf` - Generates optimized protobuf code (43% smaller than default) +3. `build:rust-client` - Builds the native Rust client binding +4. `build:ts` - Compiles TypeScript code into the build-ts directory + - When using `build:ts:release` - Applies additional optimizations with `--stripInternal` + +These steps are orchestrated by npm scripts in package.json. + +## TypeScript Build Options + +The TypeScript build uses different settings based on the build mode: + +- **Standard Build**: Uses default TypeScript settings from `tsconfig.json` +- **Release Build**: Adds the following flags: + - `--stripInternal`: Removes documentation marked with @internal from declarations while preserving public documentation + - `--pretty`: Formats error messages for better readability + - `--declaration`: Ensures type declaration files are generated with full JSDoc/TSDoc documentation + +These settings optimize the code for production use while maintaining full TypeScript type information. + +## Prerequisites + +- Node.js (see [supported version](https://github.com/valkey-io/valkey-glide/blob/main/node/README.md#nodejs-supported-version)) - npm -- git -- GCC -- pkg-config -- protoc (protobuf compiler) -- openssl -- openssl-dev -- rustup -- ziglang and zigbuild (for GNU Linux only) -- valkey (for testing) +- Rust (via `rustup`) +- `protoc` (protobuf compiler) +- GCC, pkg-config, OpenSSL -**Valkey installation** +### Installing Dependencies -See the [Valkey installation guide](https://valkey.io/topics/installation/) to install the Valkey server and CLI. +#### On Ubuntu/Debian + +```bash +sudo apt update +sudo apt install -y build-essential pkg-config libssl-dev protobuf-compiler +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +``` -**Dependencies installation for Ubuntu** +#### On macOS ```bash -sudo apt update -y -sudo apt install -y nodejs npm git gcc pkg-config protobuf-compiler openssl libssl-dev +brew install protobuf pkg-config openssl curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -source "$HOME/.cargo/env" -# Check the installed node version -node -v ``` -> **Note:** Ensure that you installed a supported Node.js version. For Ubuntu 22.04 or earlier, please refer to the instructions [here](#note-nodejs-supported-version) to upgrade your Node.js version. +### Installing Zig (for cross-compilation) + +Zig is used to enable deterministic, static builds compatible with older glibc versions like 2.17. -**Dependencies installation for CentOS** +#### On Ubuntu/Debian based systems ```bash -sudo yum update -y -sudo yum install -y nodejs git gcc pkgconfig protobuf-compiler openssl openssl-devel gettext -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -source "$HOME/.cargo/env" +sudo apt install -y snap +sudo snap install zig --classic --beta ``` -**Dependencies installation for MacOS** +#### On macOS (with Homebrew) ```bash -brew update -brew install nodejs git gcc pkgconfig protobuf openssl -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -source "$HOME/.cargo/env" +brew install zig ``` -**Install `ziglang` and `zigbuild`** +Ensure `zig` is available in your PATH: ```bash -pip3 install ziglang -cargo install --locked cargo-zigbuild +zig version ``` -#### Building and installation steps +#### Node.js and npm -Before starting this step, make sure you've installed all software requirments. +Install Node.js and npm from [nodejs.org](https://nodejs.org/) or via nvm: -1. Clone the repository: - ```bash - git clone https://github.com/valkey-io/valkey-glide.git - cd valkey-glide - ``` -2. Install all node dependencies: +```bash +nvm install --lts +nvm use --lts +``` + +Supported node version for development is minimum 18, but it is recommended to use one of the LTS versions, or latest. + +## Building & Running + +1. Clone and set up: ```bash - cd node - npm i - cd rust-client - npm i - cd .. + git clone https://github.com/valkey-io/valkey-glide.git + cd valkey-glide/node + npm install ``` -3. Build the Node wrapper (Choose a build option from the following and run it from the `node` folder): +2. Build: - 1. Build in release mode, stripped from all debug symbols (optimized and minimized binary size): +- **Fast Dev Build:** + Quickly compiles without optimization. Best for testing and development. + Command: ```bash - npm run build:release + npm run build ``` - 2. Build in release mode with debug symbols (optimized but large binary size): +- **Benchmark Build:** + Compiles optimized build, but install like dev build. + Command: ```bash npm run build:benchmark ``` - 3. For testing purposes, you can execute an unoptimized but fast build using: +- **Release Build:** + Fully optimized, stripped - This mimic production build. + Command: ```bash - npm run build + npm run build:release ``` - Once building completed, you'll find the compiled JavaScript code in the`./build-ts` folder. +## Local Development Notes -4. Run tests: +- Rust builds are located in `node/rust-client/`. Scripts automatically `cd` into this directory when running `napi build`. +- TypeScript files are compiled into the `build-ts/` directory, maintaining the modular structure. +- The `index.ts` file re-exports all public APIs to maintain a clean interface. - 1. Ensure that you have installed server and valkey-cli on your host. You can download Valkey at the following link: [Valkey Download page](https://valkey.io/download/). - 2. Execute the following command from the node folder: +## Linting & Formatting - ```bash - npm run build # make sure we have a debug build compiled first - npm test-local # For testing on local. For skipping exported symbols validation test on local. - npm test +### TypeScript - ``` +- Lint your TypeScript code using ESLint: -5. Integrating the built GLIDE package into your project: - Add the package to your project using the folder path with the command `npm install /node`. + ```bash + npm run lint + ``` -- For a fast build, execute `npm run build`. This will perform a full, unoptimized build, which is suitable for developing tests. Keep in mind that performance is significantly affected in an unoptimized build, so it's required to build with the `build:release` or `build:benchmark` option when measuring performance. -- If your modifications are limited to the TypeScript code, run `npm run build-external` to build the external package without rebuilding the internal package. -- If your modifications are limited to the Rust code, execute `npm run build-internal` to build the internal package and generate TypeScript code. -- To generate Node's protobuf files, execute `npm run build-protobuf`. Keep in mind that protobuf files are generated as part of full builds (e.g., `build`, `build:release`, etc.). +- Automatically fix linting issues: -> Note: Once building completed, you'll find the compiled JavaScript code in the `node/build-ts` folder. + ```bash + npm run lint:fix + ``` -### Troubleshooting +- Format code with Prettier: -- If the build fails after running `npx tsc` because `glide-rs` isn't found, check if your npm version is in the range 9.0.0-9.4.1, and if so, upgrade it. 9.4.2 contains a fix to a change introduced in 9.0.0 that is required in order to build the library. + ```bash + npm run format + ``` -### Test +### Rust -To run tests, use the following command: +- Run Clippy linter: -```bash -npm test -``` + ```bash + cd rust-client + cargo clippy --all-features --all-targets -- -D warnings + ``` -Simplified test suite skips few time consuming tests and runs faster: +- Format Rust code: -```bash -npm test-minimum -``` + ```bash + cargo fmt + ``` -To execute a specific test, use the [`testNamePattern`](https://jestjs.io/docs/cli#--testnamepatternregex) option with `test-dbg` script. For example: +## Testing -```bash -npm run test-dbg -- --testNamePattern="batch" -``` +- Jest configuration is in `node/jest.config.ts`. +- Tests are located in `node/tests/`. -IT suite starts the server for testing - standalone and cluster installation using `cluster_manager` script. -To run the integration tests with existing servers, run the following command: +Run tests with: ```bash -npm run test-dbg -- --cluster-endpoints=localhost:7000 --standalone-endpoints=localhost:6379 - -# If those endpoints use TLS, add `--tls=true` (applies to both endpoints) -npm run test-dbg -- --cluster-endpoints=localhost:7000 --standalone-endpoints=localhost:6379 --tls=true +npm run test ``` -Parameters `cluster-endpoints`, `standalone-endpoints` and `tls` could be used with all test suites. - -By default, the server modules tests do not run using `npm run test`. This test suite also does not start the server. -In order to run these tests, use: +Run tests in watch mode: ```bash -npm run test-modules -- --cluster-endpoints=
: +npm run test:watch ``` -Note: these tests don't run with standalone server as of now. - -### REPL (interactive shell) +### Integration and Module Tests -It is possible to run an interactive shell synced with the currect client code to test and debug it: +The project includes different test groups: ```bash -npx ts-node --project tsconfig.json -``` +# Run standard tests (excluding modules) +npm test -This shell allows executing typescript and javascript code line by line: - -```typescript -import { GlideClient, GlideClusterClient } from "."; -let client = await GlideClient.createClient({ - addresses: [{ host: "localhost", port: 6379 }], -}); -let clusterClient = await GlideClusterClient.createClient({ - addresses: [{ host: "localhost", port: 7000 }], -}); -await client.ping(); -``` +# Run tests with debugging options +npm run test:debug -After applying changes in client code you need to restart the shell. +# Run minimal tests (excludes modules and special features) +npm run test:minimum -It has command history and bash-like search (`Ctrl+R`). +# Run only module-specific tests +npm run test:modules +``` -Shell hangs on exit (`Ctrl+D`) if you don't close the clients. Use `Ctrl+C` to kill it and/or close clients before exit. +You can run these tests with custom flags for cluster and standalone endpoints by passing environment variables. -### Linters +### Package Manager and TypeScript Types Testing -Development on the Node wrapper may involve changes in either the TypeScript or Rust code. Each language has distinct linter tests that must be passed before committing changes. +The CI workflow also tests TypeScript types and package manager compatibility: -#### Language-specific Linters +- Tests proper TypeScript declaration file generation for downstream packages +- Verifies compatibility with Yarn package manager in addition to npm +- Tests library use in both direct consumer packages and transitive dependencies +- Ensures type definitions work correctly in multi-package environments -**TypeScript:** +The test structure in `node/pm-and-types-tests/` includes: -- ESLint -- Prettier +- `package1/`: A library that depends on valkey-glide +- `package2/`: An application that depends on package1 (transitive dependency on valkey-glide) -**Rust:** +This helps verify that: -- clippy -- fmt +1. Types are correctly exported and available to downstream consumers +2. The library works properly when used through different package managers +3. TypeScript type declarations are correctly passed through dependent packages -#### Running the linters +## REPL -1. TypeScript - - ```bash - # Run from the node folder - npm run lint - # To automatically apply ESLint and/or prettier recommendations - npm run lint:fix - ``` +You can experiment with the client in a live TypeScript REPL: -2. Rust - ```bash - # Run from the `node/rust-client` folder - rustup component add clippy rustfmt - cargo clippy --all-features --all-targets -- -D warnings - cargo fmt --manifest-path ./Cargo.toml --all - ``` +```bash +npx ts-node --project tsconfig.json +``` -### Recommended extensions for VS Code +## Recommended VS Code Extensions -- [Prettier - Code formatter](https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode) - JavaScript / TypeScript formatter. -- [ESLint](https://marketplace.visualstudio.com/items?itemName=dbaeumer.vscode-eslint) - linter. -- [Jest Runner](https://marketplace.visualstudio.com/items?itemName=firsttris.vscode-jest-runner) - in-editor test runner. -- [Jest Test Explorer](https://marketplace.visualstudio.com/items?itemName=kavod-io.vscode-jest-test-adapter) - adapter to the VSCode testing UI. -- [rust-analyzer](https://marketplace.visualstudio.com/items?itemName=rust-lang.rust-analyzer) - Rust language support for VSCode. +- Prettier +- ESLint +- Jest Runner +- rust-analyzer diff --git a/node/README.md b/node/README.md index 100d386980..b9928624c9 100644 --- a/node/README.md +++ b/node/README.md @@ -27,23 +27,36 @@ Refer to the [Supported Engine Versions table](https://github.com/valkey-io/valk The release of Valkey GLIDE was tested on the following platforms: -Linux: +### Linux GNU -- Ubuntu 20 (x86_64/amd64 and arm64/aarch64) -- Amazon Linux 2 (AL2) and 2023 (AL2023) (x86_64) +Tests are actively running on **Ubuntu 24**, but the package is being build for any linux using **glibc 2.17** or higher. +Manual tests have been done on: -macOS: +- Fedora 39 +- Ubuntu 22.04 +- Amazon Linux 2 +- Debian 12. -- macOS 14.7 (Apple silicon/aarch_64) -- macOS 13.7 (x86_64/amd64) +### MacOS (Darwin) -Alpine: +MacOS Apple Silicon/aarch_64 and x86_64/amd64. + +- Full tests are running on MacOS 15.0 arm64/aarch64 +- Minimal tests are running on: MacOS 13.5 x86*64/amd64 *(We do not recommend using MacOS Intel for production, It is supported for development purposes)\_ + +### Alpine + +All alpine versions that are using _musl libc_ 1.2.3 (All Alpine non deprecated alpine versions) or higher should be supported. +Tests are running on: - node:alpine (x86_64/amd64 and arm64/aarch64) ## NodeJS supported version Node.js 16 or higher. +**For npm users on linux it is recommended to use npm >=11 since it support optional download base on libc, yarn users should not be concerned** + +- Note: The library is dependent on the [protobufjs library](https://protobufjs.github.io/protobuf.js/#installation), which add a size to the package. The package is using the protobufjs/minimal version, hence, if size matter, bundlers should be able to strip the unused code. It should reduce the size of the dependency from 19kb gzipped to 6.5kb gzipped. ### Building & Testing @@ -59,7 +72,7 @@ npm i @valkey/valkey-glide ## Basic Examples -#### Standalone Mode: +#### Standalone Mode ```typescript import { GlideClient, GlideClusterClient, Logger } from "@valkey/valkey-glide"; @@ -86,7 +99,7 @@ const get_response = await client.get("foo"); console.log(`Get response is = ${get_response}`); ``` -#### Cluster Mode: +#### Cluster Mode ```typescript import { GlideClient, GlideClusterClient, Logger } from "@valkey/valkey-glide"; diff --git a/node/rust-client/.ort.yml b/node/rust-client/.ort.yml index fcafd0dd87..da86476270 100644 --- a/node/rust-client/.ort.yml +++ b/node/rust-client/.ort.yml @@ -1,11 +1,11 @@ excludes: scopes: - - pattern: 'devDependencies' - reason: 'DEV_DEPENDENCY_OF' - comment: 'Packages for development only.' - - pattern: 'build-dependencies' - reason: 'BUILD_DEPENDENCY_OF' - comment: 'Packages for building the code only.' - - pattern: 'dev-dependencies' - reason: 'DEV_DEPENDENCY_OF' - comment: 'Packages for development only.' + - pattern: "devDependencies" + reason: "DEV_DEPENDENCY_OF" + comment: "Packages for development only." + - pattern: "build-dependencies" + reason: "BUILD_DEPENDENCY_OF" + comment: "Packages for building the code only." + - pattern: "dev-dependencies" + reason: "DEV_DEPENDENCY_OF" + comment: "Packages for development only." From 2176d0bd5b8fc6390a4d61666612a483f9de1102 Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Wed, 21 May 2025 15:31:21 +0000 Subject: [PATCH 08/14] Refactor OpenTelemetry imports and streamline index exports Signed-off-by: Avi Fenesh --- eslint.config.mjs | 1 + node/src/BaseClient.ts | 3 ++- node/src/OpenTelemetry.ts | 2 +- node/src/index.ts | 20 ++------------------ node/tests/OpenTelemetry.test.ts | 2 +- 5 files changed, 7 insertions(+), 21 deletions(-) diff --git a/eslint.config.mjs b/eslint.config.mjs index 521058f5ec..b7746c7e4c 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -16,6 +16,7 @@ export default tseslint.config( "**/build-ts/**", "build/**", "jest.config.js", + "**/dist/**", "docs/**", ], }, diff --git a/node/src/BaseClient.ts b/node/src/BaseClient.ts index 6dc44316c5..149573245d 100644 --- a/node/src/BaseClient.ts +++ b/node/src/BaseClient.ts @@ -16,6 +16,8 @@ import { DEFAULT_REQUEST_TIMEOUT_IN_MILLISECONDS, Script, StartSocketConnection, + createLeakedOtelSpan, + dropOtelSpan, getStatistics, valueFromSplitPointer, } from "../build-ts/native"; @@ -261,7 +263,6 @@ import { Routes, } from "./GlideClusterClient"; import { Logger } from "./Logger"; -import { createLeakedOtelSpan, dropOtelSpan } from "./native"; import { OpenTelemetry } from "./OpenTelemetry"; /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ diff --git a/node/src/OpenTelemetry.ts b/node/src/OpenTelemetry.ts index 1ab1b3c8b0..892595e3fc 100644 --- a/node/src/OpenTelemetry.ts +++ b/node/src/OpenTelemetry.ts @@ -34,7 +34,7 @@ * - Invalid configuration will throw an error synchronously when calling `OpenTelemetry.init()`. */ -import { InitOpenTelemetry, OpenTelemetryConfig } from "glide-rs"; +import { InitOpenTelemetry, OpenTelemetryConfig } from "../build-ts/native"; import { ConfigurationError } from "./Errors"; import { Logger } from "./Logger"; diff --git a/node/src/index.ts b/node/src/index.ts index a6516c89c5..a9a44f8edf 100644 --- a/node/src/index.ts +++ b/node/src/index.ts @@ -6,24 +6,7 @@ * - All TypeScript client-side APIs from ./src/ */ -import * as Native from "../build-ts/native"; - -export const { - Level, - MAX_REQUEST_ARGS_LEN, - DEFAULT_REQUEST_TIMEOUT_IN_MILLISECONDS, - DEFAULT_CONNECTION_TIMEOUT_IN_MILLISECONDS, - DEFAULT_INFLIGHT_REQUESTS_LIMIT, - AsyncClient, - StartSocketConnection, - log, - InitInternalLogger, - valueFromSplitPointer, - createLeakedStringVec, - Script, - ClusterScanCursor, - getStatistics, -} = Native; +export * from "../build-ts/native"; // Export TypeScript APIs export * from "./BaseClient.js"; @@ -33,6 +16,7 @@ export * from "./Errors.js"; export * from "./GlideClient.js"; export * from "./GlideClusterClient.js"; export * from "./Logger.js"; +export * from "./OpenTelemetry.js"; export * from "./server-modules/GlideFt.js"; export * from "./server-modules/GlideFtOptions.js"; export * from "./server-modules/GlideJson.js"; diff --git a/node/tests/OpenTelemetry.test.ts b/node/tests/OpenTelemetry.test.ts index a375a88b81..942349e6a7 100644 --- a/node/tests/OpenTelemetry.test.ts +++ b/node/tests/OpenTelemetry.test.ts @@ -4,7 +4,6 @@ import { afterAll, afterEach, beforeAll, describe } from "@jest/globals"; import * as fs from "fs"; -import { OpenTelemetryConfig } from "glide-rs"; import { ClusterBatch, GlideClient, @@ -13,6 +12,7 @@ import { ProtocolVersion, } from ".."; import ValkeyCluster from "../../utils/TestUtils"; +import { OpenTelemetryConfig } from "../build-ts/native"; import { flushAndCloseClient, getClientConfigurationOption, From bb672dc3922c7b40b4946b8ac57f75239bfb92b1 Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Thu, 22 May 2025 12:09:48 +0000 Subject: [PATCH 09/14] fix imports Signed-off-by: Avi Fenesh --- node/src/BaseClient.ts | 61 ++++++++++------------- node/src/Batch.ts | 28 ++++------- node/src/Commands.ts | 17 +++---- node/src/GlideClient.ts | 20 ++++---- node/src/GlideClusterClient.ts | 31 ++++++------ node/src/Logger.ts | 2 +- node/src/OpenTelemetry.ts | 9 ++-- node/src/server-modules/GlideFt.ts | 8 ++- node/src/server-modules/GlideFtOptions.ts | 5 +- node/src/server-modules/GlideJson.ts | 16 ++++-- node/tests/OpenTelemetry.test.ts | 6 +-- 11 files changed, 93 insertions(+), 110 deletions(-) diff --git a/node/src/BaseClient.ts b/node/src/BaseClient.ts index 149573245d..0d41dc436b 100644 --- a/node/src/BaseClient.ts +++ b/node/src/BaseClient.ts @@ -10,22 +10,6 @@ import { Reader, Writer, } from "protobufjs/minimal"; -import { - DEFAULT_CONNECTION_TIMEOUT_IN_MILLISECONDS, - DEFAULT_INFLIGHT_REQUESTS_LIMIT, - DEFAULT_REQUEST_TIMEOUT_IN_MILLISECONDS, - Script, - StartSocketConnection, - createLeakedOtelSpan, - dropOtelSpan, - getStatistics, - valueFromSplitPointer, -} from "../build-ts/native"; -import { - command_request, - connection_request, - response, -} from "../build-ts/ProtobufMessage"; import { AggregationType, BaseScanOptions, @@ -40,8 +24,15 @@ import { BitOffsetOptions, BitwiseOperation, Boundary, + ClosingError, ClusterBatchOptions, + ConfigurationError, + ConnectionError, CoordOrigin, // eslint-disable-line @typescript-eslint/no-unused-vars + DEFAULT_CONNECTION_TIMEOUT_IN_MILLISECONDS, + DEFAULT_INFLIGHT_REQUESTS_LIMIT, + DEFAULT_REQUEST_TIMEOUT_IN_MILLISECONDS, + ExecAbortError, ExpireOptions, GeoAddOptions, GeoBoxShape, // eslint-disable-line @typescript-eslint/no-unused-vars @@ -51,20 +42,29 @@ import { GeoSearchStoreResultOptions, GeoUnit, GeospatialData, + GlideClientConfiguration, + GlideClusterClientConfiguration, HScanOptions, InsertPosition, KeyWeight, LPosOptions, ListDirection, + Logger, MemberOrigin, // eslint-disable-line @typescript-eslint/no-unused-vars + OpenTelemetry, RangeByIndex, RangeByLex, RangeByScore, + RequestError, RestoreOptions, + RouteOption, + Routes, ScoreFilter, + Script, SearchOrigin, SetOptions, SortOptions, + StartSocketConnection, StreamAddOptions, StreamClaimOptions, StreamGroupOptions, @@ -73,6 +73,8 @@ import { StreamReadOptions, StreamTrimOptions, TimeUnit, + TimeoutError, + ValkeyError, ZAddOptions, ZScanOptions, convertFieldsAndValuesToHashDataType, @@ -142,6 +144,7 @@ import { createLRem, createLSet, createLTrim, + createLeakedOtelSpan, createMGet, createMSet, createMSetNX, @@ -246,25 +249,15 @@ import { createZScore, createZUnion, createZUnionStore, -} from "./Commands"; -import { - ClosingError, - ConfigurationError, - ConnectionError, - ExecAbortError, - RequestError, - TimeoutError, - ValkeyError, -} from "./Errors"; -import { GlideClientConfiguration } from "./GlideClient"; + dropOtelSpan, + getStatistics, + valueFromSplitPointer, +} from "."; import { - GlideClusterClientConfiguration, - RouteOption, - Routes, -} from "./GlideClusterClient"; -import { Logger } from "./Logger"; -import { OpenTelemetry } from "./OpenTelemetry"; - + command_request, + connection_request, + response, +} from "../build-ts/ProtobufMessage"; /* eslint-disable-next-line @typescript-eslint/no-explicit-any */ type PromiseFunction = (value?: any) => void; type ErrorFunction = (error: ValkeyError) => void; diff --git a/node/src/Batch.ts b/node/src/Batch.ts index 2999e6b733..9f40fc5d60 100644 --- a/node/src/Batch.ts +++ b/node/src/Batch.ts @@ -2,23 +2,6 @@ * Copyright Valkey GLIDE Project Contributors - SPDX Identifier: Apache-2.0 */ -import { - ElementAndScore, - GlideRecord, - GlideString, - HashDataType, - ReadFrom, // eslint-disable-line @typescript-eslint/no-unused-vars - Score, - convertGlideRecord, -} from "./BaseClient"; - -import { - GlideClient, // eslint-disable-line @typescript-eslint/no-unused-vars -} from "./GlideClient"; -import { - GlideClusterClient, // eslint-disable-line @typescript-eslint/no-unused-vars -} from "./GlideClusterClient"; - import { AggregationType, BaseScanOptions, @@ -33,6 +16,7 @@ import { BitwiseOperation, Boundary, CoordOrigin, // eslint-disable-line @typescript-eslint/no-unused-vars + ElementAndScore, ExpireOptions, FlushMode, FunctionListOptions, @@ -47,7 +31,12 @@ import { GeoSearchStoreResultOptions, GeoUnit, GeospatialData, + GlideClient, // eslint-disable-line @typescript-eslint/no-unused-vars + GlideClusterClient, // eslint-disable-line @typescript-eslint/no-unused-vars + GlideRecord, + GlideString, HScanOptions, + HashDataType, InfoOptions, InsertPosition, KeyWeight, @@ -58,7 +47,9 @@ import { RangeByIndex, RangeByLex, RangeByScore, + ReadFrom, // eslint-disable-line @typescript-eslint/no-unused-vars RestoreOptions, + Score, ScoreFilter, SearchOrigin, SetOptions, @@ -74,6 +65,7 @@ import { ZAddOptions, ZScanOptions, convertFieldsAndValuesToHashDataType, + convertGlideRecord, createAppend, createBLMPop, createBLMove, @@ -271,7 +263,7 @@ import { createZScore, createZUnion, createZUnionStore, -} from "./Commands"; +} from "."; import { command_request } from "../build-ts/ProtobufMessage"; /** diff --git a/node/src/Commands.ts b/node/src/Commands.ts index f745948780..3d2df0629f 100644 --- a/node/src/Commands.ts +++ b/node/src/Commands.ts @@ -3,30 +3,25 @@ */ import Long from "long"; -import { - createLeakedStringVec, - MAX_REQUEST_ARGS_LEN, -} from "../build-ts/native"; - import { BaseClient, // eslint-disable-line @typescript-eslint/no-unused-vars BaseClientConfiguration, // eslint-disable-line @typescript-eslint/no-unused-vars convertRecordToGlideRecord, + createLeakedStringVec, ElementAndScore, + GlideClient, // eslint-disable-line @typescript-eslint/no-unused-vars + GlideClusterClient, // eslint-disable-line @typescript-eslint/no-unused-vars GlideRecord, GlideString, HashDataType, + MAX_REQUEST_ARGS_LEN, ObjectType, Score, + SingleNodeRoute, SortedSetDataType, -} from "./BaseClient"; +} from "."; import { command_request } from "../build-ts/ProtobufMessage"; -/* eslint-disable-next-line @typescript-eslint/no-unused-vars */ -import { GlideClient } from "./GlideClient"; -/* eslint-disable-next-line @typescript-eslint/no-unused-vars */ -import { GlideClusterClient, SingleNodeRoute } from "./GlideClusterClient"; - import RequestType = command_request.RequestType; function isLargeCommand(args: GlideString[]) { diff --git a/node/src/GlideClient.ts b/node/src/GlideClient.ts index 9338f4251b..82c6a24f59 100644 --- a/node/src/GlideClient.ts +++ b/node/src/GlideClient.ts @@ -7,17 +7,9 @@ import { AdvancedBaseClientConfiguration, BaseClient, BaseClientConfiguration, - convertGlideRecordToRecord, - Decoder, - DecoderOption, - GlideRecord, - GlideReturnType, - GlideString, - PubSubMsg, -} from "./BaseClient"; -import { Batch } from "./Batch"; -import { + Batch, BatchOptions, + convertGlideRecordToRecord, createClientGetName, createClientId, createConfigGet, @@ -52,15 +44,21 @@ import { createSelect, createTime, createUnWatch, + Decoder, + DecoderOption, FlushMode, FunctionListOptions, FunctionListResponse, FunctionRestorePolicy, FunctionStatsFullResponse, + GlideRecord, + GlideReturnType, + GlideString, InfoOptions, LolwutOptions, + PubSubMsg, ScanOptions, -} from "./Commands"; +} from "."; import { connection_request } from "../build-ts/ProtobufMessage"; /* eslint-disable-next-line @typescript-eslint/no-namespace */ diff --git a/node/src/GlideClusterClient.ts b/node/src/GlideClusterClient.ts index 29152585ff..18705d7b14 100644 --- a/node/src/GlideClusterClient.ts +++ b/node/src/GlideClusterClient.ts @@ -4,34 +4,29 @@ import * as net from "net"; import { Writer } from "protobufjs/minimal"; -import { ClusterScanCursor, Script } from "../build-ts/native"; -import { - command_request, - connection_request, -} from "../build-ts/ProtobufMessage"; import { AdvancedBaseClientConfiguration, BaseClient, BaseClientConfiguration, - Decoder, - DecoderOption, - GlideRecord, - GlideReturnType, - GlideString, - PubSubMsg, - convertGlideRecordToRecord, -} from "./BaseClient"; -import { ClusterBatch } from "./Batch"; -import { + ClusterBatch, ClusterBatchOptions, + ClusterScanCursor, ClusterScanOptions, + Decoder, + DecoderOption, FlushMode, FunctionListOptions, FunctionListResponse, FunctionRestorePolicy, FunctionStatsSingleResponse, + GlideRecord, + GlideReturnType, + GlideString, InfoOptions, LolwutOptions, + PubSubMsg, + Script, + convertGlideRecordToRecord, createClientGetName, createClientId, createConfigGet, @@ -67,7 +62,11 @@ import { createScriptKill, createTime, createUnWatch, -} from "./Commands"; +} from "."; +import { + command_request, + connection_request, +} from "../build-ts/ProtobufMessage"; /** An extension to command option types with {@link Routes}. */ export interface RouteOption { /** diff --git a/node/src/Logger.ts b/node/src/Logger.ts index 23d2fe0159..367a52fae7 100644 --- a/node/src/Logger.ts +++ b/node/src/Logger.ts @@ -2,7 +2,7 @@ * Copyright Valkey GLIDE Project Contributors - SPDX Identifier: Apache-2.0 */ -import { InitInternalLogger, Level, log } from "../build-ts/native"; +import { InitInternalLogger, Level, log } from "."; const LEVEL = new Map([ ["error", Level.Error], diff --git a/node/src/OpenTelemetry.ts b/node/src/OpenTelemetry.ts index 892595e3fc..6c82b43265 100644 --- a/node/src/OpenTelemetry.ts +++ b/node/src/OpenTelemetry.ts @@ -34,9 +34,12 @@ * - Invalid configuration will throw an error synchronously when calling `OpenTelemetry.init()`. */ -import { InitOpenTelemetry, OpenTelemetryConfig } from "../build-ts/native"; -import { ConfigurationError } from "./Errors"; -import { Logger } from "./Logger"; +import { + ConfigurationError, + InitOpenTelemetry, + Logger, + OpenTelemetryConfig, +} from "."; export class OpenTelemetry { private static _instance: OpenTelemetry | null = null; diff --git a/node/src/server-modules/GlideFt.ts b/node/src/server-modules/GlideFt.ts index 61e7c36a9f..d2487e48b0 100644 --- a/node/src/server-modules/GlideFt.ts +++ b/node/src/server-modules/GlideFt.ts @@ -9,15 +9,13 @@ import { GlideRecord, GlideReturnType, GlideString, -} from "../BaseClient"; -import { GlideClient } from "../GlideClient"; -import { GlideClusterClient } from "../GlideClusterClient"; -import { + GlideClient, + GlideClusterClient, Field, FtAggregateOptions, FtCreateOptions, FtSearchOptions, -} from "./GlideFtOptions"; +} from ".."; /** Response type of {@link GlideFt.info | ft.info} command. */ export type FtInfoReturnType = Record< diff --git a/node/src/server-modules/GlideFtOptions.ts b/node/src/server-modules/GlideFtOptions.ts index 07f70f04ce..ae43bfb7c0 100644 --- a/node/src/server-modules/GlideFtOptions.ts +++ b/node/src/server-modules/GlideFtOptions.ts @@ -2,9 +2,8 @@ * Copyright Valkey GLIDE Project Contributors - SPDX Identifier: Apache-2.0 */ -import { GlideRecord, GlideString } from "../BaseClient"; -import { SortOrder } from "../Commands"; -import { GlideFt } from "./GlideFt"; // eslint-disable-line @typescript-eslint/no-unused-vars +// eslint-disable-next-line @typescript-eslint/no-unused-vars +import { GlideFt, GlideRecord, GlideString, SortOrder } from ".."; interface BaseField { /** The name of the field. */ diff --git a/node/src/server-modules/GlideJson.ts b/node/src/server-modules/GlideJson.ts index d534ab912e..9863077e5c 100644 --- a/node/src/server-modules/GlideJson.ts +++ b/node/src/server-modules/GlideJson.ts @@ -2,11 +2,17 @@ * Copyright Valkey GLIDE Project Contributors - SPDX Identifier: Apache-2.0 */ -import { BaseClient, DecoderOption, GlideString } from "../BaseClient"; -import { Batch, ClusterBatch } from "../Batch"; -import { ConditionalChange } from "../Commands"; -import { GlideClient } from "../GlideClient"; -import { GlideClusterClient, RouteOption } from "../GlideClusterClient"; +import { + BaseClient, + Batch, + ClusterBatch, + ConditionalChange, + DecoderOption, + GlideClient, + GlideClusterClient, + GlideString, + RouteOption, +} from ".."; export type ReturnTypeJson = T | (T | null)[]; export type UniversalReturnTypeJson = T | T[]; diff --git a/node/tests/OpenTelemetry.test.ts b/node/tests/OpenTelemetry.test.ts index 942349e6a7..b4ccc2e837 100644 --- a/node/tests/OpenTelemetry.test.ts +++ b/node/tests/OpenTelemetry.test.ts @@ -4,15 +4,15 @@ import { afterAll, afterEach, beforeAll, describe } from "@jest/globals"; import * as fs from "fs"; +import ValkeyCluster from "../../utils/TestUtils"; import { ClusterBatch, GlideClient, GlideClusterClient, OpenTelemetry, + OpenTelemetryConfig, ProtocolVersion, -} from ".."; -import ValkeyCluster from "../../utils/TestUtils"; -import { OpenTelemetryConfig } from "../build-ts/native"; +} from "../build-ts"; import { flushAndCloseClient, getClientConfigurationOption, From ff9bb775e37bc41c73331ac902e749de93e9cc2b Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Thu, 22 May 2025 12:21:56 +0000 Subject: [PATCH 10/14] removing latest in favor of hard codded version Signed-off-by: Avi Fenesh --- .github/json_matrices/build-matrix.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/json_matrices/build-matrix.json b/.github/json_matrices/build-matrix.json index e6ce0b0850..aac96c33e6 100644 --- a/.github/json_matrices/build-matrix.json +++ b/.github/json_matrices/build-matrix.json @@ -2,7 +2,7 @@ { "OS": "ubuntu", "NAMED_OS": "linux", - "RUNNER": "ubuntu-latest", + "RUNNER": "ubuntu-24.04", "ARCH": "x64", "TARGET": "x86_64-unknown-linux-gnu", "PACKAGE_MANAGERS": ["pypi", "npm", "maven", "pkg_go_dev"], From 182f5df9691f4f7983616ceedb4b616a431f196c Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Thu, 22 May 2025 12:48:02 +0000 Subject: [PATCH 11/14] fix docs base on comments Signed-off-by: Avi Fenesh --- .github/DEVELOPER.md | 2 - .github/NPM-CD.md | 125 ------------- .github/workflows/node.yml | 2 +- node/DEVELOPER.md | 347 ++++++++++++++++++++----------------- 4 files changed, 186 insertions(+), 290 deletions(-) delete mode 100644 .github/NPM-CD.md diff --git a/.github/DEVELOPER.md b/.github/DEVELOPER.md index f14a2229e8..9b45af17e0 100644 --- a/.github/DEVELOPER.md +++ b/.github/DEVELOPER.md @@ -142,5 +142,3 @@ We use dynamic matrices for our CI/CD workflows, which are created using the `cr - `language-version-matrix-output`: The generated language version matrix. This dynamic matrix generation allows for flexible and efficient CI/CD workflows, adapting the test configurations based on the type of change and the specific language being tested. - -> **Note:** For detailed information about the Node.js CD workflow, see the workflow file at `.github/workflows/npm-cd.yml` diff --git a/.github/NPM-CD.md b/.github/NPM-CD.md deleted file mode 100644 index 4d18bee159..0000000000 --- a/.github/NPM-CD.md +++ /dev/null @@ -1,125 +0,0 @@ -# NPM Continuous Deployment Workflow - -This document describes the Continuous Deployment (CD) workflow for publishing Valkey Glide Node.js packages to npm. - -## Overview - -The NPM CD workflow uses a comprehensive, multi-stage approach to build, package, publish, and test the Valkey Glide Node.js packages. The workflow is defined in a single file (.github/workflows/npm-cd.yml) and includes the following benefits: - -- Comprehensive platform support (Linux with both glibc and musl, macOS) -- Cross-compilation using Zig for Linux targets (with ABI version 2.17 for maximum compatibility) -- Universal binary support for macOS (ARM64 + x86_64) -- Smart version management for different types of releases -- Thorough testing of published packages -- Optimized TypeScript build process preserving documentation -- Automatic deprecation of packages if tests fail after publishing - -## Workflow Structure - -The workflow consists of the following jobs: - -### 1. Parameter Determination (get-build-parameters) - -Determines the appropriate version number, npm tag, and platform matrix based on the workflow trigger: - -- For tags: Extracts version from the tag name (e.g., `v1.2.3` → `1.2.3`) -- For manual triggers: Uses the user-provided version -- For PRs: Uses a placeholder version (255.255.255) -- Sets npm tag based on release type: - - "latest" for stable releases (1.0.0) - - "next" for release candidates (1.0.0-rc1) -- Also determines whether packages should be published based on trigger type - -### 2. Native Module Building (build-native-modules) - -Builds the native bindings for each platform in the matrix: - -- Uses a matrix strategy to build for each supported platform -- Sets up the appropriate build environment (glibc, musl, macOS) -- Installs Rust, Zig, and other required dependencies -- Builds native Node.js modules (.node files) with napi-rs -- Uploads the native modules as artifacts for later assembly -- Uploads JS interface files (native.js/native.d.ts) as artifacts - -### 3. Package Preparation and Publishing (prepare-and-publish) - -Prepares artifacts and publishes packages (when enabled): - -- Downloads native modules and JS interface files -- Uses the NAPI artifacts command to distribute native modules to the correct package folders -- Builds TypeScript source code with optimized settings (--stripInternal --pretty --declaration) -- Sets correct versions in package.json file of the base package. -- Prepares the base package with all TypeScript code -- Copies necessary files (LICENSE, README) to each package -- Publishes the packages with single command, this single command call to prepublishOnly which is a script in package.json, napi prepublish. -- napi prepublish is a script that handles the preparation of the package for publishing, including: - - Aligning the package.json files versions with the base package version. - - Adding the native file to optionalDependencies in package.json. -- Uses npm tag based on version type (latest for stable, next for RC versions) - -### 4. Release Testing (test-published-release) - -Tests the published packages across platforms: - -- Runs a matrix strategy to test on each supported platform -- Installs Valkey (or Redis as fallback) as a prerequisite for testing -- Installs the published package from npm -- Runs utils/node tests to verify functionality -- Reports test results - -### 5. Node Tag Creation (create-node-tag) - -Creates a Node.js-specific git tag when requested: - -- Creates and pushes a tag in the format `vX.Y.Z-node` -- Only runs when explicitly requested via workflow input -- Helps track Node.js-specific releases - -### 6. Cleanup Artifacts (cleanup-artifacts) - -Cleans up artifacts after successful testing: - -- Runs after test-published-release job completes -- Removes all artifacts created during the build process to save storage space -- Deletes native module artifacts, JS files, and package artifacts - -### 7. Failure Handling (deprecate-on-failure) - -Handles failures in the publish or test process: - -- Triggers only if publishing was enabled and previous jobs failed -- Sets up Node.js and NPM with authentication -- Deprecates (not unpublishes, as valkey-glide token don't have such permissions) the base package and all platform packages to warn users -- Uses retry mechanism to handle transient errors with npm registry -- Uses robust JSON parsing to handle potential errors in package inspection -- Marks failed packages with clear deprecation notices explaining the failure - -## TypeScript Build Process - -The workflow includes an optimized TypeScript build process: - -1. Uses npm caching for faster builds -2. Installs only the necessary dependencies for building -3. Compiles TypeScript with the following optimizations: - - `--stripInternal`: Removes @internal marked items from declarations while preserving public documentation - - `--pretty`: Formats error messages for better readability - - `--declaration`: Ensures type declaration files are generated with full JSDoc/TSDoc documentation -4. Reports build statistics for monitoring output size - -## Triggers - -The workflow triggers on: - -1. Pull requests that modify: - - Workflow file itself (.github/workflows/npm-cd.yml) - - JSON matrices (.github/json_matrices/**) - - Package configuration (node/package.json) - - NPM directory structure (node/npm/**) - - Rust client files (node/rust-client/Cargo.toml, node/rust-client/src/**) - - TypeScript sources (node/src/**/*.ts) -2. Pushing tags that match the pattern "v*.*.\*" (e.g., v1.2.3, v1.2.3-rc1) - -3. Manual workflow dispatch with: - - Version input (required) - - Publish option (boolean) - - Create node tag option (boolean) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index cde481845b..6207c7baf4 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -124,7 +124,7 @@ jobs: - name: Install Yarn id: install-yarn - if: ${{ matrix.engine.version == '8.0' && matrix.host.ARCH == 'x64' && matrix.node == '23.x' }} + if: ${{ matrix.engine.version == '8.0' && matrix.host.TARGET == 'x86_64-unknown-linux-gnu' && matrix.node == '23.x' }} shell: bash run: | curl -o- -L https://yarnpkg.com/install.sh | bash diff --git a/node/DEVELOPER.md b/node/DEVELOPER.md index 64664c2aef..803e62e8d4 100644 --- a/node/DEVELOPER.md +++ b/node/DEVELOPER.md @@ -1,260 +1,283 @@ # Developer Guide -This document describes how to set up your development environment to build and test the **Valkey GLIDE Node wrapper**, part of the polyglot `valkey-glide` project. +This document describes how to set up your development environment to build and test Valkey GLIDE Node wrapper. -## Project Structure +### Development Overview -- `valkey-glide/`: The polyglot root directory. -- `valkey-glide/node/`: Contains the Node.js binding (TypeScript + Rust napi-rs). - - `src/`: TypeScript source code with modular file organization. - - `index.ts`: Main entry point that re-exports all APIs. - - `*.ts`: TypeScript files for client classes, commands, batching, etc.. - - `server-modules/`: Server module implementations (e.g., JSON, FT). - - `native.js`, `native.d.ts`: Auto-generated bindings by napi-rs. - - `build-ts/`: Compiled TypeScript output. - - `tests/`: Jest tests. - - `rust-client/`: napi-rs Rust crate (binds to `glide-core`). - - `npm/`: Platform-specific packages for CD. - - Platform-specific directories (e.g., `linux-arm64-gnu/`, `darwin-arm64/`) containing package.json and native binaries. +The GLIDE Node wrapper consists of both TypeScript and Rust code. Rust bindings for Node.js are implemented using [napi-rs](https://github.com/napi-rs/napi-rs). The Node and Rust components communicate using the [protobuf](https://github.com/protocolbuffers/protobuf) protocol. -## Development Overview +### Build from source -The Node.js wrapper consists of: +#### Prerequisites -- TypeScript: User-facing API organized in logical modules. -- Rust: Native module built with [`napi-rs`](https://github.com/napi-rs/napi-rs). -- Communication: Protocol Buffers for efficient data exchange. +Software Dependencies -### Build scripts overview +##### **Note:** Nodejs Supported Version -The build process consists of multiple steps: +If your Nodejs version is below the supported version specified in the client's [documentation](https://github.com/valkey-io/valkey-glide/blob/main/node/README.md#nodejs-supported-version), you can upgrade it using [NVM](https://github.com/nvm-sh/nvm?tab=readme-ov-file#install--update-script). -1. `clean:build` - Removes previous build artifacts -2. `build-protobuf` - Generates optimized protobuf code (43% smaller than default) -3. `build:rust-client` - Builds the native Rust client binding -4. `build:ts` - Compiles TypeScript code into the build-ts directory - - When using `build:ts:release` - Applies additional optimizations with `--stripInternal` - -These steps are orchestrated by npm scripts in package.json. - -## TypeScript Build Options - -The TypeScript build uses different settings based on the build mode: - -- **Standard Build**: Uses default TypeScript settings from `tsconfig.json` -- **Release Build**: Adds the following flags: - - `--stripInternal`: Removes documentation marked with @internal from declarations while preserving public documentation - - `--pretty`: Formats error messages for better readability - - `--declaration`: Ensures type declaration files are generated with full JSDoc/TSDoc documentation - -These settings optimize the code for production use while maintaining full TypeScript type information. - -## Prerequisites - -- Node.js (see [supported version](https://github.com/valkey-io/valkey-glide/blob/main/node/README.md#nodejs-supported-version)) - npm -- Rust (via `rustup`) -- `protoc` (protobuf compiler) -- GCC, pkg-config, OpenSSL - -### Installing Dependencies +- git +- GCC +- pkg-config +- protoc (protobuf compiler) +- openssl +- openssl-dev +- rustup -#### On Ubuntu/Debian +**Dependencies installation for Ubuntu** ```bash -sudo apt update -sudo apt install -y build-essential pkg-config libssl-dev protobuf-compiler +sudo apt update -y +sudo apt install -y nodejs npm git gcc pkg-config protobuf-compiler openssl libssl-dev curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +source "$HOME/.cargo/env" +# Check the installed node version +node -v ``` -#### On macOS +> **Note:** Ensure that you installed a supported Node.js version. For Ubuntu 22.04 or earlier, please refer to the instructions [here](#note-nodejs-supported-version) to upgrade your Node.js version. -```bash -brew install protobuf pkg-config openssl -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -``` - -### Installing Zig (for cross-compilation) - -Zig is used to enable deterministic, static builds compatible with older glibc versions like 2.17. - -#### On Ubuntu/Debian based systems - -```bash -sudo apt install -y snap -sudo snap install zig --classic --beta -``` - -#### On macOS (with Homebrew) +**Dependencies installation for CentOS** ```bash -brew install zig -``` - -Ensure `zig` is available in your PATH: - -```bash -zig version +sudo yum update -y +sudo yum install -y nodejs git gcc pkgconfig protobuf-compiler openssl openssl-devel gettext +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +source "$HOME/.cargo/env" ``` -#### Node.js and npm - -Install Node.js and npm from [nodejs.org](https://nodejs.org/) or via nvm: +**Dependencies installation for MacOS** ```bash -nvm install --lts -nvm use --lts +brew update +brew install nodejs git gcc pkgconfig protobuf openssl +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +source "$HOME/.cargo/env" ``` -Supported node version for development is minimum 18, but it is recommended to use one of the LTS versions, or latest. +**Valkey Server and CLI** +See the [Valkey installation guide](https://valkey.io/topics/installation/) to install the Valkey server and CLI. -## Building & Running +#### Building and installation steps -1. Clone and set up: +Before starting this step, make sure you've installed all software requirments. +1. Clone the repository: ```bash git clone https://github.com/valkey-io/valkey-glide.git - cd valkey-glide/node - npm install + cd valkey-glide + ``` +2. Install all node dependencies: + + ```bash + cd node + npm i + cd rust-client + npm i + cd .. ``` -2. Build: +3. Build the Node wrapper (Choose a build option from the following and run it from the `node` folder): -- **Fast Dev Build:** - Quickly compiles without optimization. Best for testing and development. - Command: + 1. Build in release mode, stripped from all debug symbols (optimized and minimized binary size): ```bash - npm run build + npm run build:release ``` -- **Benchmark Build:** - Compiles optimized build, but install like dev build. - Command: + 2. Build in release mode with debug symbols (optimized but large binary size): ```bash npm run build:benchmark ``` -- **Release Build:** - Fully optimized, stripped - This mimic production build. - Command: + 3. For testing purposes, you can execute an unoptimized but fast build using: ```bash - npm run build:release + npm run build ``` -## Local Development Notes - -- Rust builds are located in `node/rust-client/`. Scripts automatically `cd` into this directory when running `napi build`. -- TypeScript files are compiled into the `build-ts/` directory, maintaining the modular structure. -- The `index.ts` file re-exports all public APIs to maintain a clean interface. + Once building completed, you'll find the compiled JavaScript code in the`./build-ts` folder. -## Linting & Formatting +4. Run tests: -### TypeScript + 1. Ensure that you have installed server and valkey-cli on your host. You can download Valkey at the following link: [Valkey Download page](https://valkey.io/download/). + 2. Execute the following commands from the node folder: -- Lint your TypeScript code using ESLint: + ```bash + # Build first (required before testing) + npm run build - ```bash - npm run lint - ``` + # Run standard tests (excluding server modules) + npm test -- Automatically fix linting issues: + # Run tests with debugging options + npm run test:debug - ```bash - npm run lint:fix - ``` + # Run minimal tests (faster subset of tests) + npm run test:minimum -- Format code with Prettier: + # Run only server module tests (requires valkey modules) + npm run test:modules + ``` - ```bash - npm run format - ``` +5. Integrating the built GLIDE package into your project: + Add the package to your project using the folder path with the command `npm install /node`. -### Rust +- For a fast build, execute `npm run build`. This will perform a full, unoptimized build, which is suitable for developing tests. Keep in mind that performance is significantly affected in an unoptimized build, so it's required to build with the `build:release` or `build:benchmark` option when measuring performance. +- If your modifications are limited to the TypeScript code, run `npm run build:ts` to build only TypeScript code without rebuilding the Rust client. +- If your modifications are limited to the Rust code, execute `npm run build:rust-client` to build only the Rust client. +- To generate Node's protobuf files, execute `npm run build-protobuf`. Keep in mind that protobuf files are generated as part of full builds (e.g., `build`, `build:release`, etc.). -- Run Clippy linter: +> Note: Once building completed, you'll find the compiled JavaScript code in the `node/build-ts` folder. The index.ts file re-exports all APIs for a cleaner module structure. - ```bash - cd rust-client - cargo clippy --all-features --all-targets -- -D warnings - ``` +### Troubleshooting -- Format Rust code: +- If the build fails after running `npx tsc` because `glide-rs` isn't found, check if your npm version is in the range 9.0.0-9.4.1, and if so, upgrade it. 9.4.2 contains a fix to a change introduced in 9.0.0 that is required in order to build the library. - ```bash - cargo fmt - ``` +### Test -## Testing +To run tests, use the following command: -- Jest configuration is in `node/jest.config.ts`. -- Tests are located in `node/tests/`. +```bash +npm test +``` -Run tests with: +Simplified test suite skips few time consuming tests and runs faster: ```bash -npm run test +npm test-minimum ``` -Run tests in watch mode: +To execute a specific test, use the [`testNamePattern`](https://jestjs.io/docs/cli#--testnamepatternregex) option with `test-dbg` script. For example: ```bash -npm run test:watch +npm run test-dbg -- --testNamePattern="batch" ``` -### Integration and Module Tests - -The project includes different test groups: +IT suite starts the server for testing - standalone and cluster installation using `cluster_manager` script. +To run the integration tests with existing servers, run the following command: ```bash -# Run standard tests (excluding modules) -npm test +npm run test-dbg -- --cluster-endpoints=localhost:7000 --standalone-endpoints=localhost:6379 + +# If those endpoints use TLS, add `--tls=true` (applies to both endpoints) +npm run test-dbg -- --cluster-endpoints=localhost:7000 --standalone-endpoints=localhost:6379 --tls=true +``` -# Run tests with debugging options -npm run test:debug +Parameters `cluster-endpoints`, `standalone-endpoints` and `tls` could be used with all test suites. -# Run minimal tests (excludes modules and special features) -npm run test:minimum +By default, the server modules tests do not run using `npm run test`. This test suite also does not start the server. +In order to run these tests, use: -# Run only module-specific tests -npm run test:modules +```bash +npm run test-modules -- --cluster-endpoints=
: ``` -You can run these tests with custom flags for cluster and standalone endpoints by passing environment variables. +Note: these tests don't run with standalone server as of now. ### Package Manager and TypeScript Types Testing -The CI workflow also tests TypeScript types and package manager compatibility: +The project includes tests for TypeScript types and package manager compatibility: - Tests proper TypeScript declaration file generation for downstream packages - Verifies compatibility with Yarn package manager in addition to npm - Tests library use in both direct consumer packages and transitive dependencies -- Ensures type definitions work correctly in multi-package environments The test structure in `node/pm-and-types-tests/` includes: - `package1/`: A library that depends on valkey-glide -- `package2/`: An application that depends on package1 (transitive dependency on valkey-glide) +- `package2/`: An application that depends on package1 (transitive dependency) + +### REPL (interactive shell) + +It is possible to run an interactive shell synced with the currect client code to test and debug it: + +```bash +npx ts-node --project tsconfig.json +``` + +This shell allows executing typescript and javascript code line by line: + +```typescript +import { GlideClient, GlideClusterClient } from "."; +let client = await GlideClient.createClient({ + addresses: [{ host: "localhost", port: 6379 }], +}); +let clusterClient = await GlideClusterClient.createClient({ + addresses: [{ host: "localhost", port: 7000 }], +}); +await client.ping(); +``` + +After applying changes in client code you need to restart the shell. -This helps verify that: +It has command history and bash-like search (`Ctrl+R`). -1. Types are correctly exported and available to downstream consumers -2. The library works properly when used through different package managers -3. TypeScript type declarations are correctly passed through dependent packages +Shell hangs on exit (`Ctrl+D`) if you don't close the clients. Use `Ctrl+C` to kill it and/or close clients before exit. -## REPL +### Developer Utility Scripts -You can experiment with the client in a live TypeScript REPL: +Development on the Node wrapper involves various scripts for linting, formatting, and managing the development environment. + +#### Development Environment Scripts ```bash -npx ts-node --project tsconfig.json +# Launch a TypeScript REPL for interactive testing +npm run repl + +# Clean only build artifacts +npm run clean:build + +# Complete cleanup including node_modules +npm run clean +``` + +#### Linters and Formatters + +Development on the Node wrapper may involve changes in either the TypeScript or Rust code. Each language has distinct linter tests that must be passed before committing changes. + +##### TypeScript and general files + +```bash +# Run from the node folder +# Check code quality with ESLint +npm run lint + +# Automatically fix linting issues +npm run lint:fix + +# Check code formatting only +npm run prettier:check + +# Format code automatically +npm run prettier:format +``` + +##### Rust code + +```bash +# Run from the `node/rust-client` folder +# Format all files (Rust and others) +npm run format + +# Format only Rust files +npm run format:rs + +# Format only non-Rust files with Prettier +npm run format:prettier + +# Alternatively, use Rust tools directly: +rustup component add clippy rustfmt +cargo clippy --all-features --all-targets -- -D warnings +cargo fmt --manifest-path ./Cargo.toml --all ``` -## Recommended VS Code Extensions +### Recommended extensions for VS Code -- Prettier -- ESLint -- Jest Runner -- rust-analyzer +- [Prettier - Code formatter](https://marketplace.visualstudio.com/items?itemName=esbenp.prettier-vscode) - JavaScript / TypeScript formatter. +- [ESLint](https://marketplace.visualstudio.com/items?itemName=dbaeumer.vscode-eslint) - linter. +- [Jest Runner](https://marketplace.visualstudio.com/items?itemName=firsttris.vscode-jest-runner) - in-editor test runner. +- [Jest Test Explorer](https://marketplace.visualstudio.com/items?itemName=kavod-io.vscode-jest-test-adapter) - adapter to the VSCode testing UI. +- [rust-analyzer](https://marketplace.visualstudio.com/items?itemName=rust-lang.rust-analyzer) - Rust language support for VSCode. From f46552278c5cf751114f95b24ab35a8c4bb5e135 Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Thu, 22 May 2025 12:56:57 +0000 Subject: [PATCH 12/14] additional fixes of comments Signed-off-by: Avi Fenesh --- node/.ort.yml | 2 +- node/README.md | 16 +++++----------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/node/.ort.yml b/node/.ort.yml index 2adcbe1694..1a802e3cd9 100644 --- a/node/.ort.yml +++ b/node/.ort.yml @@ -4,7 +4,7 @@ excludes: reason: "DEV_DEPENDENCY_OF" comment: "Packages for development only." paths: - - pattern: "npm/glide/**" + - pattern: "npm/**" reason: "BUILD_TOOL_OF" comment: "Template file for CD" diff --git a/node/README.md b/node/README.md index b9928624c9..51db6bab87 100644 --- a/node/README.md +++ b/node/README.md @@ -29,24 +29,18 @@ The release of Valkey GLIDE was tested on the following platforms: ### Linux GNU -Tests are actively running on **Ubuntu 24**, but the package is being build for any linux using **glibc 2.17** or higher. -Manual tests have been done on: - -- Fedora 39 -- Ubuntu 22.04 -- Amazon Linux 2 -- Debian 12. +Linux with **glibc 2.17** or higher. ### MacOS (Darwin) MacOS Apple Silicon/aarch_64 and x86_64/amd64. - Full tests are running on MacOS 15.0 arm64/aarch64 -- Minimal tests are running on: MacOS 13.5 x86*64/amd64 *(We do not recommend using MacOS Intel for production, It is supported for development purposes)\_ +- Minimal tests are running on: MacOS 13.5 x86*64/amd64*(We do not recommend using MacOS Intel for production, It is supported for development purposes)\_ ### Alpine -All alpine versions that are using _musl libc_ 1.2.3 (All Alpine non deprecated alpine versions) or higher should be supported. +All alpine versions that are using *musl libc* 1.2.3 (All Alpine non deprecated alpine versions) or higher should be supported. Tests are running on: - node:alpine (x86_64/amd64 and arm64/aarch64) @@ -72,7 +66,7 @@ npm i @valkey/valkey-glide ## Basic Examples -#### Standalone Mode +#### Standalone Mode: ```typescript import { GlideClient, GlideClusterClient, Logger } from "@valkey/valkey-glide"; @@ -99,7 +93,7 @@ const get_response = await client.get("foo"); console.log(`Get response is = ${get_response}`); ``` -#### Cluster Mode +#### Cluster Mode: ```typescript import { GlideClient, GlideClusterClient, Logger } from "@valkey/valkey-glide"; From 899b5ab38865205eca890519dbc80dec711a0a70 Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Sun, 25 May 2025 11:18:05 +0000 Subject: [PATCH 13/14] address pr comments Signed-off-by: Avi Fenesh --- .github/workflows/node.yml | 92 +++++++++++++++---- .github/workflows/npm-cd.yml | 53 +---------- node/DEVELOPER.md | 4 +- .../.yarnrc.yml | 0 .../package.json | 2 +- .../src/index.ts | 0 .../tsconfig.json | 0 .../.yarnrc.yml | 0 .../package.json | 0 .../src/index.ts | 0 .../src/types.ts | 0 .../tsconfig.json | 0 12 files changed, 76 insertions(+), 75 deletions(-) rename node/pm-and-types-tests/{package1 => depend-on-glide-dependent}/.yarnrc.yml (100%) rename node/pm-and-types-tests/{package2 => depend-on-glide-dependent}/package.json (88%) rename node/pm-and-types-tests/{package2 => depend-on-glide-dependent}/src/index.ts (100%) rename node/pm-and-types-tests/{package2 => depend-on-glide-dependent}/tsconfig.json (100%) rename node/pm-and-types-tests/{package2 => depend-on-glide-package}/.yarnrc.yml (100%) rename node/pm-and-types-tests/{package1 => depend-on-glide-package}/package.json (100%) rename node/pm-and-types-tests/{package1 => depend-on-glide-package}/src/index.ts (100%) rename node/pm-and-types-tests/{package1 => depend-on-glide-package}/src/types.ts (100%) rename node/pm-and-types-tests/{package1 => depend-on-glide-package}/tsconfig.json (100%) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 6207c7baf4..d39a847433 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -115,8 +115,72 @@ jobs: ${{ matrix.host.TARGET }}-glide-core ${{ matrix.host.TARGET }} - # Install npm dependencies - - name: Build + # Install npm dependencies and build with release flag to ensure we testing the release version + - name: Install and Build + working-directory: ./node + run: | + npm install + npm run build:release + + - name: Test + shell: bash + run: | + npm run build + npm run test + working-directory: ./node + env: + JEST_HTML_REPORTER_OUTPUT_PATH: test-report-node.html + + - name: Upload test reports + if: always() + continue-on-error: true + uses: actions/upload-artifact@v4 + with: + name: node-test-reports-${{ matrix.host.OS }}-${{ matrix.host.ARCH }}-${{ matrix.node }}-${{ github.run_id }} + path: | + node/test-report*.html + + test-node-extras: + name: Node Extra Tests (ES/CJS, Yarn, Types, Benchmarks) + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + + - name: Output job information + run: | + echo "Running one-time Node.js extra tests" + echo "Target: x86_64-unknown-linux-gnu" + echo "Node: 23.x" + echo "Engine: 8.0" + + # Install software dependencies + - name: Install shared dependencies + uses: ./.github/workflows/install-shared-dependencies + with: + os: ubuntu + target: x86_64-unknown-linux-gnu + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: "8.0" + + # Setup Node.js + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 23.x + + - uses: actions/cache@v4 + with: + path: | + node/rust-client/target + glide-core/src/generated + key: x86_64-unknown-linux-gnu-node-extras + restore-keys: | + x86_64-unknown-linux-gnu-glide-core + x86_64-unknown-linux-gnu + + # Install npm dependencies and build with release flag + - name: Install and Build working-directory: ./node run: | npm install @@ -124,10 +188,10 @@ jobs: - name: Install Yarn id: install-yarn - if: ${{ matrix.engine.version == '8.0' && matrix.host.TARGET == 'x86_64-unknown-linux-gnu' && matrix.node == '23.x' }} shell: bash run: | curl -o- -L https://yarnpkg.com/install.sh | bash + - name: Add Yarn to PATH id: add-yarn-to-path if: ${{ steps.install-yarn.outcome == 'success' }} @@ -140,18 +204,17 @@ jobs: working-directory: ./node/pm-and-types-tests shell: bash run: | - cd package1 + cd depend-on-glide-package yarn install rm -rf node_modules/@valkey/valkey-glide/build-ts/* cp -r ../../build-ts ./node_modules/@valkey/valkey-glide/ yarn run build - cd ../package2 + cd ../depend-on-glide-dependant yarn install yarn run build - # Additional tests for specific configurations + # Test hybrid node modules - commonjs - name: Test hybrid node modules - commonjs - if: ${{ matrix.engine.version == '8.0' && matrix.node == '23.x' && matrix.host.TARGET == 'x86_64-unknown-linux-gnu' }} run: | npm install npm run test @@ -160,7 +223,6 @@ jobs: JEST_HTML_REPORTER_OUTPUT_PATH: test-report-commonjs.html - name: Test hybrid node modules - ecma - if: ${{ matrix.engine.version == '8.0' && matrix.node == '23.x' && matrix.host.TARGET == 'x86_64-unknown-linux-gnu' }} run: | npm install npm run test @@ -170,27 +232,17 @@ jobs: - name: Run benchmarks uses: ./.github/workflows/test-benchmark - if: ${{ matrix.engine.version == '8.0' && matrix.node == '23.x' && matrix.host.TARGET == 'x86_64-unknown-linux-gnu' }} with: language-flag: -node - - name: Test - shell: bash - run: | - npm run build - npm run test - working-directory: ./node - env: - JEST_HTML_REPORTER_OUTPUT_PATH: test-report-node.html - - name: Upload test reports if: always() continue-on-error: true uses: actions/upload-artifact@v4 with: - name: node-test-reports-${{ matrix.host.OS }}-${{ matrix.host.ARCH }}-${{ matrix.node }}-${{ github.run_id }} + name: node-extra-test-reports-${{ github.run_id }} path: | - node/test-report*.html + node/hybrid-node-tests/**/test-report*.html utils/clusters/** benchmarks/results/** diff --git a/.github/workflows/npm-cd.yml b/.github/workflows/npm-cd.yml index a1d737f2b9..bf16dea8a5 100644 --- a/.github/workflows/npm-cd.yml +++ b/.github/workflows/npm-cd.yml @@ -25,11 +25,6 @@ on: required: true type: boolean default: false - create_node_tag: - description: "Create git tag with -node suffix" - required: false - type: boolean - default: false concurrency: group: node-cd-${{ github.head_ref || github.ref }}-${{ toJson(inputs) }} @@ -47,7 +42,6 @@ jobs: npm_tag: ${{ steps.get-params.outputs.npm_tag }} platform_matrix: ${{ steps.load-platform-matrix.outputs.platform_matrix }} should_publish: ${{ steps.get-params.outputs.should_publish }} - should_create_node_tag: ${{ steps.get-params.outputs.should_create_node_tag }} steps: - name: Checkout uses: actions/checkout@v4 @@ -72,10 +66,8 @@ jobs: run: | RELEASE="255.255.255" SHOULD_PUB="false" - SHOULD_TAG="false" echo "release_version=$RELEASE" >> $GITHUB_OUTPUT echo "should_publish=$SHOULD_PUB" >> $GITHUB_OUTPUT - echo "should_create_node_tag=$SHOULD_TAG" >> $GITHUB_OUTPUT echo "Pull request detected - using development version $RELEASE" - name: Handle workflow dispatch event @@ -85,7 +77,6 @@ jobs: env: INPUT_VER: ${{ github.event.inputs.version }} INPUT_PUB: ${{ github.event.inputs.publish }} - INPUT_TAG: ${{ github.event.inputs.create_node_tag }} run: | # Validate version exists if [[ -z "$INPUT_VER" ]]; then @@ -100,11 +91,9 @@ jobs: # All good - set variables RELEASE="$INPUT_VER" SHOULD_PUB="$INPUT_PUB" - SHOULD_TAG="$INPUT_TAG" echo "release_version=$RELEASE" >> $GITHUB_OUTPUT echo "should_publish=$SHOULD_PUB" >> $GITHUB_OUTPUT - echo "should_create_node_tag=$SHOULD_TAG" >> $GITHUB_OUTPUT - echo "Manual workflow run - using version $RELEASE (publish=$SHOULD_PUB, tag=$SHOULD_TAG)" + echo "Manual workflow run - using version $RELEASE (publish=$SHOULD_PUB)" - name: Handle tag push event id: tag-push-params @@ -114,10 +103,8 @@ jobs: if [[ "${{ steps.event-check.outputs.ref }}" =~ ^refs/tags/v([0-9]+\.[0-9]+\.[0-9]+(-rc[0-9]+)?)$ ]]; then RELEASE="${BASH_REMATCH[1]}" SHOULD_PUB="true" - SHOULD_TAG="false" # Auto tag creation only for manual workflow runs echo "release_version=$RELEASE" >> $GITHUB_OUTPUT echo "should_publish=$SHOULD_PUB" >> $GITHUB_OUTPUT - echo "should_create_node_tag=$SHOULD_TAG" >> $GITHUB_OUTPUT echo "Tag push detected - using version $RELEASE from tag" else echo "::error::Invalid tag format. Expected vX.Y.Z or vX.Y.Z-rcN"; exit 1 @@ -148,18 +135,15 @@ jobs: RELEASE_VERSION: ${{ steps.pull-request-params.outputs.release_version || steps.workflow-dispatch-params.outputs.release_version || steps.tag-push-params.outputs.release_version }} NPM_TAG: ${{ steps.determine-npm-tag.outputs.npm_tag }} SHOULD_PUBLISH: ${{ steps.pull-request-params.outputs.should_publish || steps.workflow-dispatch-params.outputs.should_publish || steps.tag-push-params.outputs.should_publish }} - SHOULD_CREATE_NODE_TAG: ${{ steps.pull-request-params.outputs.should_create_node_tag || steps.workflow-dispatch-params.outputs.should_create_node_tag || steps.tag-push-params.outputs.should_create_node_tag }} run: | echo "release_version=$RELEASE_VERSION" >> $GITHUB_OUTPUT echo "npm_tag=$NPM_TAG" >> $GITHUB_OUTPUT echo "should_publish=$SHOULD_PUBLISH" >> $GITHUB_OUTPUT - echo "should_create_node_tag=$SHOULD_CREATE_NODE_TAG" >> $GITHUB_OUTPUT echo "Final build parameters:" echo "- Version: $RELEASE_VERSION" echo "- NPM Tag: $NPM_TAG" echo "- Publish: $SHOULD_PUBLISH" - echo "- Create Node Tag: $SHOULD_CREATE_NODE_TAG" - name: Load Platform Matrix id: load-platform-matrix @@ -467,41 +451,6 @@ jobs: artifacts-* js-files-${{ github.run_id }} - create-node-tag: - needs: [get-build-parameters, test-published-release] - # Only create the tag if the publish was successful, tests passed, and the tag creation is requested - if: | - needs.get-build-parameters.outputs.should_publish == 'true' && - needs.get-build-parameters.outputs.should_create_node_tag == 'true' - permissions: - contents: write - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Create and push Node tag - shell: bash - env: - VERSION: ${{ needs.get-build-parameters.outputs.release_version }} - run: | - # Create node version tag - NODE_TAG="v${VERSION}-node" - echo "Creating node tag: ${NODE_TAG}" - - # Configure git - git config --global user.name "GitHub Actions" - git config --global user.email "actions@github.com" - - # Create and push the tag - git tag -a "${NODE_TAG}" -m "Node.js release ${VERSION}" - git push origin "${NODE_TAG}" - - echo "Successfully created and pushed tag: ${NODE_TAG}" - test-published-release: needs: [get-build-parameters, prepare-and-publish] # Only run tests if the publish was successful diff --git a/node/DEVELOPER.md b/node/DEVELOPER.md index 803e62e8d4..d924abbbd8 100644 --- a/node/DEVELOPER.md +++ b/node/DEVELOPER.md @@ -187,8 +187,8 @@ The project includes tests for TypeScript types and package manager compatibilit The test structure in `node/pm-and-types-tests/` includes: -- `package1/`: A library that depends on valkey-glide -- `package2/`: An application that depends on package1 (transitive dependency) +- `depend-on-glide-package/`: A library that depends on valkey-glide +- `depend-on-glide-dependent/`: An application that depends on depend-on-glide-package (transitive dependency) ### REPL (interactive shell) diff --git a/node/pm-and-types-tests/package1/.yarnrc.yml b/node/pm-and-types-tests/depend-on-glide-dependent/.yarnrc.yml similarity index 100% rename from node/pm-and-types-tests/package1/.yarnrc.yml rename to node/pm-and-types-tests/depend-on-glide-dependent/.yarnrc.yml diff --git a/node/pm-and-types-tests/package2/package.json b/node/pm-and-types-tests/depend-on-glide-dependent/package.json similarity index 88% rename from node/pm-and-types-tests/package2/package.json rename to node/pm-and-types-tests/depend-on-glide-dependent/package.json index 766923ad6e..c97723db7e 100644 --- a/node/pm-and-types-tests/package2/package.json +++ b/node/pm-and-types-tests/depend-on-glide-dependent/package.json @@ -9,7 +9,7 @@ "dev": "ts-node src/index.ts" }, "dependencies": { - "@test/common": "file:../package1", + "@test/common": "file:../depend-on-glide-package", "winston": "^3.17.0" }, "devDependencies": { diff --git a/node/pm-and-types-tests/package2/src/index.ts b/node/pm-and-types-tests/depend-on-glide-dependent/src/index.ts similarity index 100% rename from node/pm-and-types-tests/package2/src/index.ts rename to node/pm-and-types-tests/depend-on-glide-dependent/src/index.ts diff --git a/node/pm-and-types-tests/package2/tsconfig.json b/node/pm-and-types-tests/depend-on-glide-dependent/tsconfig.json similarity index 100% rename from node/pm-and-types-tests/package2/tsconfig.json rename to node/pm-and-types-tests/depend-on-glide-dependent/tsconfig.json diff --git a/node/pm-and-types-tests/package2/.yarnrc.yml b/node/pm-and-types-tests/depend-on-glide-package/.yarnrc.yml similarity index 100% rename from node/pm-and-types-tests/package2/.yarnrc.yml rename to node/pm-and-types-tests/depend-on-glide-package/.yarnrc.yml diff --git a/node/pm-and-types-tests/package1/package.json b/node/pm-and-types-tests/depend-on-glide-package/package.json similarity index 100% rename from node/pm-and-types-tests/package1/package.json rename to node/pm-and-types-tests/depend-on-glide-package/package.json diff --git a/node/pm-and-types-tests/package1/src/index.ts b/node/pm-and-types-tests/depend-on-glide-package/src/index.ts similarity index 100% rename from node/pm-and-types-tests/package1/src/index.ts rename to node/pm-and-types-tests/depend-on-glide-package/src/index.ts diff --git a/node/pm-and-types-tests/package1/src/types.ts b/node/pm-and-types-tests/depend-on-glide-package/src/types.ts similarity index 100% rename from node/pm-and-types-tests/package1/src/types.ts rename to node/pm-and-types-tests/depend-on-glide-package/src/types.ts diff --git a/node/pm-and-types-tests/package1/tsconfig.json b/node/pm-and-types-tests/depend-on-glide-package/tsconfig.json similarity index 100% rename from node/pm-and-types-tests/package1/tsconfig.json rename to node/pm-and-types-tests/depend-on-glide-package/tsconfig.json From 6fefec84ae4a2f9ca5acd6d9d60e1b87ac7e4446 Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Sun, 25 May 2025 11:38:11 +0000 Subject: [PATCH 14/14] typo Signed-off-by: Avi Fenesh --- .github/workflows/node.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index d39a847433..7f13fc8b88 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -209,7 +209,7 @@ jobs: rm -rf node_modules/@valkey/valkey-glide/build-ts/* cp -r ../../build-ts ./node_modules/@valkey/valkey-glide/ yarn run build - cd ../depend-on-glide-dependant + cd ../depend-on-glide-dependent yarn install yarn run build