diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index a3d82cf2..7940324d 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -28,7 +28,7 @@ global_job_config: - git submodule update --init --recursive - cd deps/librdkafka - git fetch origin - - git checkout v2.5.3 + - git checkout v2.6.0 - cd ../../ - cache clear @@ -70,7 +70,8 @@ blocks: jobs: - name: "Build from source and test" commands: - - npm install # this will actually not build anything if we have a release, but rather, fetch things using node-pre-gyp - so change this later. + - export CKJS_LINKING=dynamic + - npm run install-from-source - make test - name: 'macOS arm64/m1: Build and test' @@ -82,7 +83,8 @@ blocks: jobs: - name: 'Build from source and test' commands: - - npm install # this will actually not build anything if we have a release, but rather, fetch things using node-pre-gyp - so change this later. + - export CKJS_LINKING=dynamic + - npm run install-from-source - make test - name: "Linux amd64: Build, test, lint" @@ -93,7 +95,8 @@ blocks: type: s1-prod-ubuntu20-04-amd64-2 prologue: commands: - - npm install # this will actually not build anything if we have a release, but rather, fetch things using node-pre-gyp - so change this later. + - export CKJS_LINKING=dynamic + - npm run install-from-source jobs: - name: "Test" commands: @@ -130,10 +133,7 @@ blocks: - sudo apt-get install -y librdkafka-dev bc - export CKJS_LINKING=dynamic - export BUILD_LIBRDKAFKA=0 - - npm install - - npx node-pre-gyp --build-from-source clean - - npx node-pre-gyp --build-from-source configure - - npx node-pre-gyp --build-from-source build + - npm run install-from-source jobs: - name: "Performance Test" commands: @@ -168,7 +168,7 @@ blocks: - sem-version node 18.19.0 - export NODE_ABI=108 - export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz" - - npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet. + - npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet. - npx node-pre-gyp package - ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY} - artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" @@ -177,7 +177,7 @@ blocks: - sem-version node 20.10.0 - export NODE_ABI=115 - export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz" - - npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet. + - npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet. - npx node-pre-gyp package - ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY} - artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" @@ -186,7 +186,7 @@ blocks: - sem-version node 21.4.0 - export NODE_ABI=120 - export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz" - - npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet. + - npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet. - npx node-pre-gyp package - ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY} - artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" @@ -195,7 +195,7 @@ blocks: - sem-version node 22.2.0 - export NODE_ABI=127 - export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz" - - npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet. + - npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet. - npx node-pre-gyp package - ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY} - artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" @@ -224,7 +224,7 @@ blocks: - sem-version node 18.19.0 - export NODE_ABI=108 - export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz" - - npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet. + - npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet. - npx node-pre-gyp package - ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY} - artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" @@ -233,7 +233,7 @@ blocks: - sem-version node 20.10.0 - export NODE_ABI=115 - export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz" - - npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet. + - npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet. - npx node-pre-gyp package - ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY} - artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" @@ -242,7 +242,7 @@ blocks: - sem-version node 21.4.0 - export NODE_ABI=120 - export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz" - - npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet. + - npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet. - npx node-pre-gyp package - ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY} - artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" @@ -251,7 +251,7 @@ blocks: - sem-version node 22.2.0 - export NODE_ABI=127 - export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz" - - npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet. + - npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet. - npx node-pre-gyp package - ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY} - artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" @@ -373,7 +373,7 @@ blocks: - sem-version node 18.19.0 - export NODE_ABI=108 - export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz" - - npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet. + - npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet. - npx node-pre-gyp package - ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY} - artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" @@ -382,7 +382,7 @@ blocks: - sem-version node 20.10.0 - export NODE_ABI=115 - export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz" - - npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet. + - npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet. - npx node-pre-gyp package - ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY} - artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" @@ -391,7 +391,7 @@ blocks: - sem-version node 21.4.0 - export NODE_ABI=120 - export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz" - - npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet. + - npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet. - npx node-pre-gyp package - ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY} - artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" @@ -400,7 +400,7 @@ blocks: - sem-version node 22.2.0 - export NODE_ABI=127 - export ARTIFACT_KEY="confluent-kafka-javascript-${SEMAPHORE_GIT_TAG_NAME}-node-v${NODE_ABI}-${PLATFORM}-${LIBC}-${ARCHITECTURE}.tar.gz" - - npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet. + - npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet. - npx node-pre-gyp package - ls build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY} - artifact push workflow "build/stage/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" --destination "releases/${SEMAPHORE_GIT_TAG_NAME}/${ARTIFACT_KEY}" @@ -450,7 +450,7 @@ blocks: - $env:NODE_ABI = 108 - $env:ARTIFACT_KEY = "confluent-kafka-javascript-${env:SEMAPHORE_GIT_TAG_NAME}-node-v${env:NODE_ABI}-${env:PLATFORM}-${env:LIBC}-${env:ARCHITECTURE}.tar.gz" - echo "$env:ARTIFACT_KEY" - - npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet. + - npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet. - npx node-pre-gyp package - ls "build/stage/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}" - artifact push workflow "build/stage/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}" --destination "releases/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}" @@ -463,7 +463,7 @@ blocks: - $env:NODE_ABI = 115 - $env:ARTIFACT_KEY = "confluent-kafka-javascript-${env:SEMAPHORE_GIT_TAG_NAME}-node-v${env:NODE_ABI}-${env:PLATFORM}-${env:LIBC}-${env:ARCHITECTURE}.tar.gz" - echo "$env:ARTIFACT_KEY" - - npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet. + - npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet. - npx node-pre-gyp package - ls "build/stage/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}" - artifact push workflow "build/stage/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}" --destination "releases/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}" @@ -476,7 +476,7 @@ blocks: - $env:NODE_ABI = 120 - $env:ARTIFACT_KEY = "confluent-kafka-javascript-${env:SEMAPHORE_GIT_TAG_NAME}-node-v${env:NODE_ABI}-${env:PLATFORM}-${env:LIBC}-${env:ARCHITECTURE}.tar.gz" - echo "$env:ARTIFACT_KEY" - - npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet. + - npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet. - npx node-pre-gyp package - ls "build/stage/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}" - artifact push workflow "build/stage/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}" --destination "releases/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}" @@ -489,7 +489,7 @@ blocks: - $env:NODE_ABI = 127 - $env:ARTIFACT_KEY = "confluent-kafka-javascript-${env:SEMAPHORE_GIT_TAG_NAME}-node-v${env:NODE_ABI}-${env:PLATFORM}-${env:LIBC}-${env:ARCHITECTURE}.tar.gz" - echo "$env:ARTIFACT_KEY" - - npm install # node-pre-gyp will fallback to build here, because new tag implies no release yet. + - npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet. - npx node-pre-gyp package - ls "build/stage/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}" - artifact push workflow "build/stage/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}" --destination "releases/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}" diff --git a/deps/librdkafka b/deps/librdkafka index 9416dd80..7fc5a598 160000 --- a/deps/librdkafka +++ b/deps/librdkafka @@ -1 +1 @@ -Subproject commit 9416dd80fb0dba71ff73a8cb4d2b919f54651006 +Subproject commit 7fc5a59826ee1e1dee3236b1e43180cc22cfb496 diff --git a/deps/librdkafka.gyp b/deps/librdkafka.gyp index 94057d96..a08912ed 100644 --- a/deps/librdkafka.gyp +++ b/deps/librdkafka.gyp @@ -32,7 +32,7 @@ "deps/librdkafka/src/librdkafka.so", ], "action": [ - "make", "-C", "librdkafka", "libs", "install" + "make", "-j", "-C", "librdkafka", "libs", "install" ], "conditions": [ [ diff --git a/lib/error.js b/lib/error.js index ccdc263e..600c6715 100644 --- a/lib/error.js +++ b/lib/error.js @@ -27,7 +27,7 @@ LibrdKafkaError.wrap = errorWrap; * @enum {number} * @constant */ -// ====== Generated from librdkafka master file src-cpp/rdkafkacpp.h ====== +// ====== Generated from librdkafka 2.6.0 file src-cpp/rdkafkacpp.h ====== LibrdKafkaError.codes = { /* Internal errors to rdkafka: */ diff --git a/package.json b/package.json index 92b9d0a0..fbb0ad7d 100644 --- a/package.json +++ b/package.json @@ -2,8 +2,8 @@ "name": "@confluentinc/kafka-javascript", "version": "v0.2.1", "description": "Node.js bindings for librdkafka", - "librdkafka": "2.5.3", - "librdkafka_win": "2.5.3", + "librdkafka": "2.6.0", + "librdkafka_win": "2.6.0", "main": "lib/index.js", "types": "types/index.d.ts", "scripts": { @@ -11,6 +11,7 @@ "build": "node-gyp build", "test": "make test", "install": "node-pre-gyp install --fallback-to-build", + "install-from-source": "npm ci && node-pre-gyp install --build-from-source=@confluentinc/kafka-javascript --fallback-to-build", "prepack": "node ./ci/prepublish.js", "test:types": "tsc -p ." }, diff --git a/types/config.d.ts b/types/config.d.ts index f0b1f708..493ac807 100644 --- a/types/config.d.ts +++ b/types/config.d.ts @@ -1,4 +1,4 @@ -// ====== Generated from librdkafka master file CONFIGURATION.md ====== +// ====== Generated from librdkafka 2.6.0 file CONFIGURATION.md ====== // Code that generated this is a derivative work of the code from Nam Nguyen // https://gist.github.com/ntgn81/066c2c8ec5b4238f85d1e9168a04e3fb @@ -620,12 +620,33 @@ export interface GlobalConfig { "client.rack"?: string; /** - * Controls how the client uses DNS lookups. By default, when the lookup returns multiple IP addresses for a hostname, they will all be attempted for connection before the connection is considered failed. This applies to both bootstrap and advertised servers. If the value is set to `resolve_canonical_bootstrap_servers_only`, each entry will be resolved and expanded into a list of canonical names. NOTE: Default here is different from the Java client's default behavior, which connects only to the first IP address returned for a hostname. + * The backoff time in milliseconds before retrying a protocol request, this is the first backoff time, and will be backed off exponentially until number of retries is exhausted, and it's capped by retry.backoff.max.ms. + * + * @default 100 + */ + "retry.backoff.ms"?: number; + + /** + * The max backoff time in milliseconds before retrying a protocol request, this is the atmost backoff allowed for exponentially backed off requests. + * + * @default 1000 + */ + "retry.backoff.max.ms"?: number; + + /** + * Controls how the client uses DNS lookups. By default, when the lookup returns multiple IP addresses for a hostname, they will all be attempted for connection before the connection is considered failed. This applies to both bootstrap and advertised servers. If the value is set to `resolve_canonical_bootstrap_servers_only`, each entry will be resolved and expanded into a list of canonical names. **WARNING**: `resolve_canonical_bootstrap_servers_only` must only be used with `GSSAPI` (Kerberos) as `sasl.mechanism`, as it's the only purpose of this configuration value. **NOTE**: Default here is different from the Java client's default behavior, which connects only to the first IP address returned for a hostname. * * @default use_all_dns_ips */ "client.dns.lookup"?: 'use_all_dns_ips' | 'resolve_canonical_bootstrap_servers_only'; + /** + * Whether to enable pushing of client metrics to the cluster, if the cluster has a client metrics subscription which matches this client + * + * @default true + */ + "enable.metrics.push"?: boolean; + /** * Enables or disables `event.*` emitting. * @@ -703,20 +724,6 @@ export interface ProducerGlobalConfig extends GlobalConfig { */ "retries"?: number; - /** - * The backoff time in milliseconds before retrying a protocol request, this is the first backoff time, and will be backed off exponentially until number of retries is exhausted, and it's capped by retry.backoff.max.ms. - * - * @default 100 - */ - "retry.backoff.ms"?: number; - - /** - * The max backoff time in milliseconds before retrying a protocol request, this is the atmost backoff allowed for exponentially backed off requests. - * - * @default 1000 - */ - "retry.backoff.max.ms"?: number; - /** * The threshold of outstanding not yet transmitted broker requests needed to backpressure the producer's message accumulator. If the number of not yet transmitted requests equals or exceeds this number, produce request creation that would have otherwise been triggered (for example, in accordance with linger.ms) will be delayed. A lower number yields larger and more effective batches. A higher value can improve latency when using compression on slow machines. * @@ -762,7 +769,7 @@ export interface ProducerGlobalConfig extends GlobalConfig { /** * Delivery report callback (set with rd_kafka_conf_set_dr_cb()) */ - "dr_cb"?: boolean | ((...args: any[]) => any); + "dr_cb"?: boolean | Function; /** * Delivery report callback (set with rd_kafka_conf_set_dr_msg_cb()) @@ -810,12 +817,24 @@ export interface ConsumerGlobalConfig extends GlobalConfig { "heartbeat.interval.ms"?: number; /** - * Group protocol type for the `generic` group protocol. NOTE: Currently, the only supported group protocol type is `consumer`. + * Group protocol type for the `classic` group protocol. NOTE: Currently, the only supported group protocol type is `consumer`. * * @default consumer */ "group.protocol.type"?: string; + /** + * Group protocol to use. Use `classic` for the original protocol and `consumer` for the new protocol introduced in KIP-848. Available protocols: classic or consumer. Default is `classic`, but will change to `consumer` in next releases. + * + * @default classic + */ + "group.protocol"?: 'classic' | 'consumer'; + + /** + * Server side assignor to use. Keep it null to make server select a suitable assignor for the group. Available assignors: uniform or range. Default is null + */ + "group.remote.assignor"?: string; + /** * How often to query for the current client group coordinator. If the currently assigned coordinator is down the configured query interval will be divided by ten to more quickly recover in case of coordinator reassignment. * @@ -936,12 +955,12 @@ export interface ConsumerGlobalConfig extends GlobalConfig { /** * Called after consumer group has been rebalanced (set with rd_kafka_conf_set_rebalance_cb()) */ - "rebalance_cb"?: boolean | ((...args: any[]) => any); + "rebalance_cb"?: boolean | Function; /** * Offset commit result propagation callback. (set with rd_kafka_conf_set_offset_commit_cb()) */ - "offset_commit_cb"?: boolean | ((...args: any[]) => any); + "offset_commit_cb"?: boolean | Function; /** * Emit RD_KAFKA_RESP_ERR__PARTITION_EOF event whenever the consumer reaches the end of a partition. diff --git a/types/errors.d.ts b/types/errors.d.ts index 333b17dc..5af09927 100644 --- a/types/errors.d.ts +++ b/types/errors.d.ts @@ -1,4 +1,4 @@ -// ====== Generated from librdkafka master file src-cpp/rdkafkacpp.h ====== +// ====== Generated from librdkafka 2.6.0 file src-cpp/rdkafkacpp.h ====== export const CODES: { ERRORS: { /* Internal errors to rdkafka: */ /** Begin internal error codes (**-200**) */