From 494f43613cd8e721aef8789fc1c3059329a714e1 Mon Sep 17 00:00:00 2001 From: Travis Thompson Date: Wed, 12 Jun 2019 17:15:58 -0700 Subject: [PATCH 001/114] Bring dockerfile up to date --- crypto-algos.txt | 1925 ++++++++++++++++++++++++++++++++ distribution/docker/Dockerfile | 31 +- distribution/pom.xml | 70 ++ 3 files changed, 2004 insertions(+), 22 deletions(-) create mode 100644 crypto-algos.txt diff --git a/crypto-algos.txt b/crypto-algos.txt new file mode 100644 index 000000000000..071300bb46a3 --- /dev/null +++ b/crypto-algos.txt @@ -0,0 +1,1925 @@ + + +┌───────────────────┐ +│ 251 Code Findings │ +└───────────────────┘ + +  core/src/main/java/org/apache/druid/common/utils/SocketUtil.java  + infosec-fips-rules.java.unencrypted-socket + Detected use of network without encryption. + +  ▶▶┆ Autofix ▶ s/(.*)/// Appears that this is using the network without encryption, please + verify and fix. https://go/fips-compliance + \1 + /1 + 44┆ try (ServerSocket socket = new ServerSocket(currPort)) { + +  core/src/main/java/org/apache/druid/crypto/CryptoService.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 120┆ Cipher ecipher = Cipher.getInstance(transformation); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 144┆ Cipher dcipher = Cipher.getInstance(transformation); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 156┆ SecretKeyFactory factory = SecretKeyFactory.getInstance(secretKeyFactoryAlg); + +  core/src/main/java/org/apache/druid/guice/Binders.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 55┆ return PolyBind.optionBinder(binder, Key.get(DataSegmentPusher.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 60┆ return PolyBind.optionBinder(binder, Key.get(TaskLogs.class)); + +  core/src/main/java/org/apache/druid/guice/DruidSecondaryModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 72┆ binder.bind(ObjectMapper.class).to(Key.get(ObjectMapper.class, Json.class)); + +  core/src/main/java/org/apache/druid/guice/GuiceAnnotationIntrospector.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 61┆ return Key.get(m.getGenericType()); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 63┆ return Key.get(m.getGenericType(), guiceAnnotation); + +  core/src/main/java/org/apache/druid/guice/JacksonConfigProvider.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 38┆ binder.bind(Key.get(Types.newParameterizedType(Supplier.class, clazz))) + +  core/src/main/java/org/apache/druid/guice/JsonConfigProvider.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 91┆ Key.get(classToProvide), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 92┆ (Key) Key.get(Types.newParameterizedType(Supplier.class, classToProvide)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 109┆ Key.get(classToProvide), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 110┆ (Key) Key.get(Types.newParameterizedType(Supplier.class, classToProvide)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 121┆ Key.get(classToProvide, annotation), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 122┆ (Key) Key.get(Types.newParameterizedType(Supplier.class, classToProvide), annotation) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 138┆ Key.get(classToProvide, annotation), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 139┆ (Key) Key.get(Types.newParameterizedType(Supplier.class, classToProvide), annotation) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 181┆ supplierKey = Key.get(supType, bindKey.getAnnotationType()); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 183┆ supplierKey = Key.get(supType, bindKey.getAnnotation()); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 185┆ supplierKey = Key.get(supType); + +  core/src/main/java/org/apache/druid/guice/LifecycleModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 71┆ registerKey(binder, Key.get(clazz)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 96┆ registerKey(binder, Key.get(clazz, annotation)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 141┆ final Key> keyHolderKey = Key.get(new TypeLiteral>(){}, + Names.named("lifecycle")); + +  core/src/main/java/org/apache/druid/guice/ListProvider.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 40┆ return add(Key.get(clazz)); + +  core/src/main/java/org/apache/druid/guice/PolyBind.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 178┆ implsMap = (Map>) injector.getInstance(Key.get(mapType, + key.getAnnotation())); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 180┆ implsMap = (Map>) injector.getInstance(Key.get(mapType, + key.getAnnotationType())); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 182┆ implsMap = (Map>) injector.getInstance(Key.get(mapType)); + +  core/src/main/java/org/apache/druid/java/util/http/client/HttpClientInit.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 111┆ final KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType()); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 111┆ final KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType()); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 117┆ final SSLContext sslContext = SSLContext.getInstance("TLS"); + ⋮┆---------------------------------------- + infosec-fips-rules.java.java.lang.security.audit.weak-ssl-context.weak-ssl-context + An insecure SSL context was detected. TLS versions 1.0, 1.1, and all SSL versions are + considered weak encryption and are deprecated. Use SSLContext.getInstance("TLSv1.2") for the + best security. + +  ▶▶┆ Autofix ▶ s/(.*)/// A minimum version of TLS1.2 is required for FIPS compliance. Please + review. https://go/fips-compliance + \1 + /1 + 117┆ final SSLContext sslContext = SSLContext.getInstance("TLS"); + +  + extensions-contrib/ambari-metrics-emitter/src/main/java/org/apache/druid/emitter/ambari/metrics/AmbariMetricsEmitterMo + dule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 62┆ .map((String name) -> injector.getInstance(Key.get(Emitter.class, Names.named(name)))) + +  + extensions-contrib/cassandra-storage/src/main/java/org/apache/druid/storage/cassandra/CassandraDruidModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 44┆ PolyBind.optionBinder(binder, Key.get(DataSegmentPusher.class)) + +  + extensions-contrib/dropwizard-emitter/src/main/java/org/apache/druid/emitter/dropwizard/DropwizardEmitterModule.java +  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 65┆ Key.get( + 66┆  Emitter.class, + 67┆  Names.named(s) + 68┆ ))) + +  + extensions-contrib/graphite-emitter/src/main/java/org/apache/druid/emitter/graphite/GraphiteEmitterModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 65┆ return injector.getInstance(Key.get(Emitter.class, Names.named(alertEmitterName))); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 74┆ return injector.getInstance(Key.get(Emitter.class, Names.named(requestLogEmitterName))); + +  extensions-contrib/influxdb-emitter/src/main/java/org/apache/druid/emitter/influxdb/InfluxdbEmitter.java +  + infosec-fips-rules.java.insecure-hostname-verifier + Insecure HostnameVerifier implementation detected. This will accept any SSL certificate with + any hostname, which creates the possibility for man-in-the-middle attacks. + +  ▶▶┆ Autofix ▶ s/(.*)/// This Insecure HostnameVerifier will accept any SSL certificate without + a hostname, which is insecure. Please review. https://go/fips-compliance + \1 + /1 + 31┆ import org.apache.http.conn.ssl.NoopHostnameVerifier; + ⋮┆---------------------------------------- + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 233┆ KeyStore store = KeyStore.getInstance(influxdbEmitterConfig.getTrustStoreType()); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 237┆ sslContext = SSLContext.getInstance("TLS"); + ⋮┆---------------------------------------- + infosec-fips-rules.java.java.lang.security.audit.weak-ssl-context.weak-ssl-context + An insecure SSL context was detected. TLS versions 1.0, 1.1, and all SSL versions are + considered weak encryption and are deprecated. Use SSLContext.getInstance("TLSv1.2") for the + best security. + +  ▶▶┆ Autofix ▶ s/(.*)/// A minimum version of TLS1.2 is required for FIPS compliance. Please + review. https://go/fips-compliance + \1 + /1 + 237┆ sslContext = SSLContext.getInstance("TLS"); + +  + extensions-contrib/influxdb-emitter/src/main/java/org/apache/druid/emitter/influxdb/InfluxdbEmitterConfig.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 91┆ this.trustStoreType = trustStoreType == null ? KeyStore.getDefaultType() : trustStoreType; + +  + extensions-contrib/kubernetes-overlord-extensions/src/main/java/org/apache/druid/k8s/overlord/K8sOverlordModule.java +  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 52┆ Key.get(TaskRunnerFactory.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 53┆ Key.get(KubernetesTaskRunnerFactory.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 57┆ Key.get(TaskRunnerFactory.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 69┆ PolyBind.createChoice(binder, "druid.indexer.logs.type", Key.get(TaskLogs.class), + Key.get(FileTaskLogs.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 69┆ PolyBind.createChoice(binder, "druid.indexer.logs.type", Key.get(TaskLogs.class), + Key.get(FileTaskLogs.class)); + +  + extensions-contrib/sqlserver-metadata-storage/src/main/java/org/apache/druid/metadata/storage/sqlserver/SQLServerMetad + ataStorageModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 62┆ .optionBinder(binder, Key.get(MetadataStorageProvider.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 68┆ .optionBinder(binder, Key.get(MetadataStorageConnector.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 74┆ .optionBinder(binder, Key.get(SQLMetadataConnector.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 79┆ PolyBind.optionBinder(binder, Key.get(MetadataStorageActionHandlerFactory.class)) + +  extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/BasicAuthUtils.java +  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 108┆ SecretKeyFactory keyFactory = SecretKeyFactory.getInstance(ALGORITHM); + +  + extensions-core/druid-basic-security/src/main/java/org/apache/druid/security/basic/BasicSecurityDruidModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 251┆ serviceName = injector.getInstance(Key.get(String.class, Names.named("serviceName"))); + +  + extensions-core/kubernetes-extensions/src/main/java/org/apache/druid/k8s/discovery/K8sDiscoveryModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 78┆ PolyBind.optionBinder(binder, Key.get(DruidNodeDiscoveryProvider.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 83┆ PolyBind.optionBinder(binder, Key.get(DruidNodeAnnouncer.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 88┆ PolyBind.optionBinder(binder, Key.get(DruidLeaderSelector.class, Coordinator.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 95┆ PolyBind.optionBinder(binder, Key.get(DruidLeaderSelector.class, IndexingService.class)) + +  + extensions-core/lookups-cached-global/src/main/java/org/apache/druid/server/lookup/namespace/NamespaceExtractionModule + .java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 84┆ .createChoiceWithDefault(binder, TYPE_PREFIX, Key.get(NamespaceExtractionCacheManager.class), + "onHeap") + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 88┆ .optionBinder(binder, Key.get(NamespaceExtractionCacheManager.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 94┆ .optionBinder(binder, Key.get(NamespaceExtractionCacheManager.class)) + +  extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/MSQTasks.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 124┆ return injector.getInstance(Key.get(StorageConnector.class, MultiStageQuery.class)); + +  extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/guice/MSQDurableStorageModule.java +  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 86┆ binder.bind(Key.get(StorageConnector.class, MultiStageQuery.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 87┆ .toProvider(Key.get(StorageConnectorProvider.class, MultiStageQuery.class)) + +  + extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/IndexerControllerContext.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 82┆ return injector.getInstance(Key.get(DruidNode.class, Self.class)); + +  extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/IndexerWorkerContext.java +  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 101┆ injector.getInstance(Key.get(ServiceClientFactory.class, EscalatedGlobal.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 243┆ return injector.getInstance(Key.get(DruidNode.class, Self.class)); + +  extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/indexing/MSQControllerTask.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 182┆ injector.getInstance(Key.get(ServiceClientFactory.class, EscalatedGlobal.class)); + +  + extensions-core/mysql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/mysql/MySQLMetadataStorageModul + e.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 72┆ .optionBinder(binder, Key.get(MetadataStorageProvider.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 78┆ .optionBinder(binder, Key.get(MetadataStorageConnector.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 84┆ .optionBinder(binder, Key.get(SQLMetadataConnector.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 89┆ PolyBind.optionBinder(binder, Key.get(MetadataStorageActionHandlerFactory.class)) + +  + extensions-core/postgresql-metadata-storage/src/main/java/org/apache/druid/metadata/storage/postgresql/PostgreSQLMetad + ataStorageModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 73┆ .optionBinder(binder, Key.get(MetadataStorageProvider.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 79┆ .optionBinder(binder, Key.get(MetadataStorageConnector.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 85┆ .optionBinder(binder, Key.get(SQLMetadataConnector.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 90┆ PolyBind.optionBinder(binder, Key.get(MetadataStorageActionHandlerFactory.class)) + +  indexing-hadoop/src/main/java/org/apache/druid/indexer/HadoopDruidIndexerConfig.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 115┆ Key.get(DruidNode.class, Self.class), + +  indexing-service/src/main/java/org/apache/druid/guice/IndexingServiceTaskLogsModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 40┆ PolyBind.createChoice(binder, "druid.indexer.logs.type", Key.get(TaskLogs.class), + Key.get(FileTaskLogs.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 40┆ PolyBind.createChoice(binder, "druid.indexer.logs.type", Key.get(TaskLogs.class), + Key.get(FileTaskLogs.class)); + +  indexing-service/src/main/java/org/apache/druid/indexing/common/IndexTaskClient.java  + infosec-fips-rules.java.unencrypted-socket + Detected use of network without encryption. + +  ▶▶┆ Autofix ▶ s/(.*)/// Appears that this is using the network without encryption, please + verify and fix. https://go/fips-compliance + \1 + /1 + 208┆ new Socket(host, port).close(); + +  indexing-service/src/main/java/org/apache/druid/indexing/overlord/PortFinder.java  + infosec-fips-rules.java.unencrypted-socket + Detected use of network without encryption. + +  ▶▶┆ Autofix ▶ s/(.*)/// Appears that this is using the network without encryption, please + verify and fix. https://go/fips-compliance + \1 + /1 + 50┆ new ServerSocket(portNum).close(); + +  integration-tests/src/main/java/org/apache/druid/cli/CliCustomNodeRole.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 102┆ LifecycleModule.registerKey(binder, Key.get(SelfDiscoveryResource.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 129┆ final ObjectMapper jsonMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class)); + +  integration-tests-ex/tools/src/main/java/org/apache/druid/testing/tools/CliCustomNodeRole.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 103┆ LifecycleModule.registerKey(binder, Key.get(SelfDiscoveryResource.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 129┆ final ObjectMapper jsonMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class)); + +  processing/src/main/java/org/apache/druid/jackson/JacksonModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 46┆ binder.bind(ObjectMapper.class).to(Key.get(ObjectMapper.class, Json.class)); + +  processing/src/main/java/org/apache/druid/query/aggregation/JavaScriptAggregatorFactory.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 255┆ MessageDigest md = MessageDigest.getInstance("SHA-1"); + +  server/src/main/java/org/apache/druid/client/cache/MemcachedCache.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 416┆ SSLContext sslContext = SSLContext.getInstance("TLS"); + ⋮┆---------------------------------------- + infosec-fips-rules.java.java.lang.security.audit.weak-ssl-context.weak-ssl-context + An insecure SSL context was detected. TLS versions 1.0, 1.1, and all SSL versions are + considered weak encryption and are deprecated. Use SSLContext.getInstance("TLSv1.2") for the + best security. + +  ▶▶┆ Autofix ▶ s/(.*)/// A minimum version of TLS1.2 is required for FIPS compliance. Please + review. https://go/fips-compliance + \1 + /1 + 416┆ SSLContext sslContext = SSLContext.getInstance("TLS"); + +  server/src/main/java/org/apache/druid/curator/discovery/DiscoveryModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 110┆ registerKey(binder, Key.get(new TypeLiteral(){})); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 124┆ registerKey(binder, Key.get(new TypeLiteral(){}, annotation)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 139┆ registerKey(binder, Key.get(new TypeLiteral(){}, annotation)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 170┆ .to(Key.get(CuratorServiceAnnouncer.class, Names.named(NAME))) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 173┆ binder.bind(Key.get(ServiceAnnouncer.Noop.class, Names.named(NAME))).toInstance(new + ServiceAnnouncer.Noop()); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 175┆ .to(Key.get(ServiceAnnouncer.Noop.class, Names.named(NAME))) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 180┆ PolyBind.createChoiceWithDefault(binder, INTERNAL_DISCOVERY_PROP, + Key.get(DruidNodeAnnouncer.class), CURATOR_KEY); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 185┆ Key.get(DruidNodeDiscoveryProvider.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 192┆ Key.get(DruidLeaderSelector.class, Coordinator.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 199┆ Key.get(DruidLeaderSelector.class, IndexingService.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 203┆ PolyBind.optionBinder(binder, Key.get(DruidNodeDiscoveryProvider.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 208┆ PolyBind.optionBinder(binder, Key.get(DruidNodeAnnouncer.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 213┆ PolyBind.optionBinder(binder, Key.get(DruidLeaderSelector.class, Coordinator.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 222┆ PolyBind.optionBinder(binder, Key.get(DruidLeaderSelector.class, IndexingService.class)) + +  server/src/main/java/org/apache/druid/guice/CacheModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 51┆ binder.bind(Cache.class).toProvider(Key.get(CacheProvider.class, + Global.class)).in(ManageLifecycle.class); + +  server/src/main/java/org/apache/druid/guice/DruidInjectorBuilder.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 75┆ this.jsonMapper = baseInjector.getInstance(Key.get(ObjectMapper.class, Json.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 76┆ this.smileMapper = baseInjector.getInstance(Key.get(ObjectMapper.class, Smile.class)); + +  server/src/main/java/org/apache/druid/guice/LocalDataStorageDruidModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 61┆ Key.get(DataSegmentPusher.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 62┆ Key.get(LocalDataSegmentPusher.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 68┆ Key.get(DataSegmentKiller.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 69┆ Key.get(LocalDataSegmentKiller.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 85┆ PolyBind.optionBinder(binder, Key.get(DataSegmentPusher.class)) + +  server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 75┆ .toProvider(Key.get(QuerySchedulerProvider.class, Global.class)) + +  server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 110┆ Key.get(GenericQueryMetricsFactory.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 111┆ Key.get(DefaultGenericQueryMetricsFactory.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 114┆ .optionBinder(binder, Key.get(GenericQueryMetricsFactory.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 121┆ Key.get(GroupByQueryMetricsFactory.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 122┆ Key.get(DefaultGroupByQueryMetricsFactory.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 125┆ .optionBinder(binder, Key.get(GroupByQueryMetricsFactory.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 132┆ Key.get(TimeseriesQueryMetricsFactory.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 133┆ Key.get(DefaultTimeseriesQueryMetricsFactory.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 136┆ .optionBinder(binder, Key.get(TimeseriesQueryMetricsFactory.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 143┆ Key.get(TopNQueryMetricsFactory.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 144┆ Key.get(DefaultTopNQueryMetricsFactory.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 147┆ .optionBinder(binder, Key.get(TopNQueryMetricsFactory.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 154┆ Key.get(SearchQueryMetricsFactory.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 155┆ Key.get(DefaultSearchQueryMetricsFactory.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 158┆ .optionBinder(binder, Key.get(SearchQueryMetricsFactory.class)) + +  server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 72┆ PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataStorageConnector.class), + defaultValue); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 73┆ PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataStorageProvider.class), + defaultValue); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 74┆ PolyBind.createChoiceWithDefault(binder, prop, Key.get(SQLMetadataConnector.class), + defaultValue); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 76┆ PolyBind.createChoiceWithDefault(binder, prop, Key.get(SegmentsMetadataManager.class), + defaultValue); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 77┆ PolyBind.createChoiceWithDefault(binder, prop, Key.get(SegmentsMetadataManagerProvider.class), + defaultValue); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 78┆ PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataRuleManager.class), + defaultValue); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 79┆ PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataRuleManagerProvider.class), + defaultValue); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 80┆ PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataSegmentPublisher.class), + defaultValue); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 81┆ PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataSegmentPublisherProvider.class), + defaultValue); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 82┆ PolyBind.createChoiceWithDefault(binder, prop, + Key.get(IndexerMetadataStorageCoordinator.class), defaultValue); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 83┆ PolyBind.createChoiceWithDefault(binder, prop, + Key.get(MetadataStorageActionHandlerFactory.class), defaultValue); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 84┆ PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataStorageUpdaterJobHandler.class), + defaultValue); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 85┆ PolyBind.createChoiceWithDefault(binder, prop, Key.get(AuditManager.class), defaultValue); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 86┆ PolyBind.createChoiceWithDefault(binder, prop, Key.get(AuditManagerProvider.class), + defaultValue); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 87┆ PolyBind.createChoiceWithDefault(binder, prop, Key.get(MetadataSupervisorManager.class), + defaultValue); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 93┆ PolyBind.optionBinder(binder, Key.get(SegmentsMetadataManager.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 98┆ PolyBind.optionBinder(binder, Key.get(SegmentsMetadataManagerProvider.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 103┆ PolyBind.optionBinder(binder, Key.get(MetadataRuleManager.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 108┆ PolyBind.optionBinder(binder, Key.get(MetadataRuleManagerProvider.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 113┆ PolyBind.optionBinder(binder, Key.get(MetadataSegmentPublisher.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 118┆ PolyBind.optionBinder(binder, Key.get(MetadataSegmentPublisherProvider.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 123┆ PolyBind.optionBinder(binder, Key.get(IndexerMetadataStorageCoordinator.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 128┆ PolyBind.optionBinder(binder, Key.get(MetadataStorageUpdaterJobHandler.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 135┆ PolyBind.optionBinder(binder, Key.get(AuditManager.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 140┆ PolyBind.optionBinder(binder, Key.get(AuditManagerProvider.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 145┆ PolyBind.optionBinder(binder, Key.get(MetadataSupervisorManager.class)) + +  server/src/main/java/org/apache/druid/guice/http/AbstractHttpClientProvider.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 49┆ configKey = Key.get( + 50┆  new TypeLiteral>() + 51┆  { + 52┆  }, annotation + 53┆ ); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 54┆ sslContextKey = Key.get(SSLContext.class, annotation); + +  server/src/main/java/org/apache/druid/guice/security/AuthenticatorModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 41┆ Key.get(Authenticator.class) + +  server/src/main/java/org/apache/druid/guice/security/AuthorizerModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 41┆ Key.get(Authorizer.class) + +  server/src/main/java/org/apache/druid/initialization/Log4jShutterDownerModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 77┆ binder.bind(Key.get(Log4jShutterDowner.class, Names.named("ForTheEagerness"))) + +  server/src/main/java/org/apache/druid/metadata/storage/derby/DerbyMetadataStorageDruidModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 52┆ PolyBind.optionBinder(binder, Key.get(MetadataStorageProvider.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 57┆ PolyBind.optionBinder(binder, Key.get(MetadataStorageConnector.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 62┆ PolyBind.optionBinder(binder, Key.get(SQLMetadataConnector.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 67┆ PolyBind.optionBinder(binder, Key.get(MetadataStorageActionHandlerFactory.class)) + +  server/src/main/java/org/apache/druid/server/emitter/ComposingEmitterModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 65┆ return injector.getInstance(Key.get(Emitter.class, Names.named(s))); + +  server/src/main/java/org/apache/druid/server/emitter/HttpEmitterModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 69┆ context = SSLContext.getDefault(); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 122┆ effectiveSSLContext = SSLContext.getDefault(); + +  server/src/main/java/org/apache/druid/server/initialization/jetty/ChatHandlerServerModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 78┆ binder.bind(DruidNode.class).annotatedWith(RemoteChatHandler.class).to(Key.get(DruidNode.class, + Self.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 79┆ + binder.bind(ServerConfig.class).annotatedWith(RemoteChatHandler.class).to(Key.get(ServerConfig.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 80┆ + binder.bind(TLSServerConfig.class).annotatedWith(RemoteChatHandler.class).to(Key.get(TLSServerConfig.class)) + ; + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 109┆ injector.getExistingBinding(Key.get(SslContextFactory.Server.class)), + +  server/src/main/java/org/apache/druid/server/initialization/jetty/CliIndexerServerModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 104┆ binder.bind(DruidNode.class).annotatedWith(RemoteChatHandler.class).to(Key.get(DruidNode.class, + Self.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 105┆ + binder.bind(ServerConfig.class).annotatedWith(RemoteChatHandler.class).to(Key.get(ServerConfig.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 106┆ + binder.bind(TLSServerConfig.class).annotatedWith(RemoteChatHandler.class).to(Key.get(TLSServerConfig.class)) + ; + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 135┆ injector.getExistingBinding(Key.get(SslContextFactory.Server.class)), + +  server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerInitUtils.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 65┆ injector.getInstance(Key.get(new TypeLiteral>() {})); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 72┆ injector.getInstance(Key.get(new TypeLiteral>() {})); + +  server/src/main/java/org/apache/druid/server/initialization/jetty/JettyServerModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 184┆ injector.getExistingBinding(Key.get(SslContextFactory.Server.class)), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 313┆ ? KeyStore.getDefaultType() + +  server/src/main/java/org/apache/druid/server/metrics/MetricsModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 95┆ binder.bind(Key.get(MonitorScheduler.class, Names.named("ForTheEagerness"))) + +  server/src/main/java/org/apache/druid/server/security/TLSUtils.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 184┆ sslContext = SSLContext.getInstance(protocol == null ? "TLSv1.2" : protocol); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 185┆ KeyStore trustStore = KeyStore.getInstance(trustStoreType == null + 186┆  ? KeyStore.getDefaultType() + 187┆  : trustStoreType); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 186┆ ? KeyStore.getDefaultType() + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 201┆ KeyStore keyStore = KeyStore.getInstance(keyStoreType == null + 202┆  ? KeyStore.getDefaultType() + 203┆  : keyStoreType); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 202┆ ? KeyStore.getDefaultType() + +  services/src/main/java/org/apache/druid/cli/CliBroker.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 171┆ LifecycleModule.registerKey(binder, Key.get(SelfDiscoveryResource.class)); + +  services/src/main/java/org/apache/druid/cli/CliCoordinator.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 322┆ LifecycleModule.registerKey(binder, Key.get(SelfDiscoveryResource.class)); + +  services/src/main/java/org/apache/druid/cli/CliHistorical.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 138┆ LifecycleModule.registerKey(binder, Key.get(SelfDiscoveryResource.class)); + +  services/src/main/java/org/apache/druid/cli/CliIndexer.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 186┆ LifecycleModule.registerKey(binder, Key.get(SelfDiscoveryResource.class)); + +  services/src/main/java/org/apache/druid/cli/CliMiddleManager.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 145┆ Key.get(RowIngestionMetersFactory.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 146┆ Key.get(DropwizardRowIngestionMetersFactory.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 149┆ PolyBind.optionBinder(binder, Key.get(RowIngestionMetersFactory.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 174┆ LifecycleModule.registerKey(binder, Key.get(SelfDiscoveryResource.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 184┆ Key.get(IntermediaryDataManager.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 185┆ Key.get(LocalIntermediaryDataManager.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 189┆ Key.get(IntermediaryDataManager.class) + +  services/src/main/java/org/apache/druid/cli/CliOverlord.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 227┆ Key.get(RowIngestionMetersFactory.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 228┆ Key.get(DropwizardRowIngestionMetersFactory.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 231┆ PolyBind.optionBinder(binder, Key.get(RowIngestionMetersFactory.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 276┆ LifecycleModule.registerKey(binder, Key.get(SelfDiscoveryResource.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 286┆ Key.get(TaskStorage.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 287┆ Key.get(HeapMemoryTaskStorage.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 290┆ PolyBind.optionBinder(binder, Key.get(TaskStorage.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 303┆ Key.get(IntermediaryDataManager.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 304┆ Key.get(LocalIntermediaryDataManager.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 308┆ Key.get(IntermediaryDataManager.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 321┆ Key.get(TaskRunnerFactory.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 322┆ Key.get(HttpRemoteTaskRunnerFactory.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 326┆ Key.get(TaskRunnerFactory.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 359┆ Key.get(ProvisioningStrategy.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 360┆ Key.get(SimpleWorkerProvisioningStrategy.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 364┆ Key.get(ProvisioningStrategy.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 413┆ final ObjectMapper jsonMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class)); + +  services/src/main/java/org/apache/druid/cli/CliPeon.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 354┆ Key.get(RowIngestionMetersFactory.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 355┆ Key.get(DropwizardRowIngestionMetersFactory.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 358┆ PolyBind.optionBinder(binder, Key.get(RowIngestionMetersFactory.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 371┆ Key.get(ChatHandlerProvider.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 372┆ Key.get(ServiceAnnouncingChatHandlerProvider.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 375┆ PolyBind.optionBinder(binder, Key.get(ChatHandlerProvider.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 404┆ Key.get(TaskActionClientFactory.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 405┆ Key.get(RemoteTaskActionClientFactory.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 408┆ PolyBind.optionBinder(binder, Key.get(TaskActionClientFactory.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 470┆ Key.get(IntermediaryDataManager.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 471┆ Key.get(LocalIntermediaryDataManager.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 475┆ Key.get(IntermediaryDataManager.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 483┆ Key.get(ShuffleClient.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 484┆ Key.get(HttpShuffleClient.class) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 488┆ Key.get(ShuffleClient.class) + +  services/src/main/java/org/apache/druid/cli/CliRouter.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 128┆ LifecycleModule.registerKey(binder, Key.get(SelfDiscoveryResource.class)); + +  services/src/main/java/org/apache/druid/cli/CoordinatorJettyServerInitializer.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 81┆ final ObjectMapper jsonMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class)); + +  services/src/main/java/org/apache/druid/cli/CreateTables.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 83┆ Key.get(MetadataStorageConnectorConfig.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 107┆ Key.get(MetadataStorageTablesConfig.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 112┆ Key.get(DruidNode.class, Self.class), + +  services/src/main/java/org/apache/druid/cli/DumpSegment.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 233┆ final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 280┆ final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, + Json.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 360┆ final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, + Json.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 444┆ final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, + Json.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 560┆ final ObjectMapper objectMapper = injector.getInstance(Key.get(ObjectMapper.class, + Json.class)); + +  services/src/main/java/org/apache/druid/cli/ExportMetadata.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 148┆ Key.get(MetadataStorageConnectorConfig.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 172┆ Key.get(MetadataStorageTablesConfig.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 177┆ Key.get(DruidNode.class, Self.class), + +  services/src/main/java/org/apache/druid/cli/MiddleManagerJettyServerInitializer.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 72┆ final ObjectMapper jsonMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class)); + +  services/src/main/java/org/apache/druid/cli/QueryJettyServerInitializer.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 98┆ final ObjectMapper jsonMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class)); + +  services/src/main/java/org/apache/druid/cli/ResetCluster.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 90┆ Key.get(DruidNode.class, Self.class), + +  services/src/main/java/org/apache/druid/cli/RouterJettyServerInitializer.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 118┆ final ObjectMapper jsonMapper = injector.getInstance(Key.get(ObjectMapper.class, Json.class)); + +  services/src/main/java/org/apache/druid/cli/ServerRunnable.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 84┆ LifecycleModule.registerKey(binder, Key.get(DiscoverySideEffectsProvider.Child.class)); + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 98┆ LifecycleModule.registerKey(binder, Key.get(DiscoverySideEffectsProvider.Child.class, + annotation)); + +  services/src/main/java/org/apache/druid/guice/AbstractDruidServiceModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 79┆ .to(Key.get(new TypeLiteral>>(){}, + role.getDruidServiceInjectName())); + +  sql/src/main/java/org/apache/druid/sql/calcite/aggregation/SqlAggregationModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 52┆ Key.get(SqlAggregator.class, ApproxCountDistinct.class), + +  sql/src/main/java/org/apache/druid/sql/guice/SqlBindings.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 57┆ PolyBind.optionBinder(binder, Key.get(SqlAggregator.class, ApproxCountDistinct.class)) + +  sql/src/main/java/org/apache/druid/sql/guice/SqlModule.java  + infosec-fips-rules.java.detect-crypto-usage + This catchall rule detects the use of any cryptographic function for review + +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 80┆ PolyBind.optionBinder(binder, Key.get(ViewManager.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 88┆ Key.get(ViewManager.class), + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 92┆ PolyBind.optionBinder(binder, Key.get(DruidSchemaManager.class)) + ⋮┆---------------------------------------- +  ▶▶┆ Autofix ▶ s/(.*)/// Detected the use of a crypographic function. Please review this for + compliance. https://go/fips-compliance + \1 + /1 + 100┆ Key.get(DruidSchemaManager.class), + diff --git a/distribution/docker/Dockerfile b/distribution/docker/Dockerfile index 2bcd28f873a6..3cda10b09b3d 100644 --- a/distribution/docker/Dockerfile +++ b/distribution/docker/Dockerfile @@ -18,16 +18,9 @@ # ARG JDK_VERSION=17 +FROM alpine as extractor -# The platform is explicitly specified as x64 to build the Druid distribution. -# This is because it's not able to build the distribution on arm64 due to dependency problem of web-console. See: https://github.com/apache/druid/issues/13012 -# Since only java jars are shipped in the final image, it's OK to build the distribution on x64. -# Once the web-console dependency problem is resolved, we can remove the --platform directive. -FROM --platform=linux/amd64 maven:3.8.6-jdk-11-slim as builder - -# Rebuild from source in this stage -# This can be unset if the tarball was already built outside of Docker -ARG BUILD_FROM_SOURCE="true" +ARG VERSION RUN export DEBIAN_FRONTEND=noninteractive \ && apt-get -qq update \ @@ -35,19 +28,10 @@ RUN export DEBIAN_FRONTEND=noninteractive \ COPY . /src WORKDIR /src -RUN --mount=type=cache,target=/root/.m2 if [ "$BUILD_FROM_SOURCE" = "true" ]; then \ - mvn -B -ff -q dependency:go-offline \ - install \ - -Pdist,bundle-contrib-exts \ - -Pskip-static-checks,skip-tests \ - -Dmaven.javadoc.skip=true \ - ; fi - -RUN --mount=type=cache,target=/root/.m2 VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate \ - -Dexpression=project.version -DforceStdout=true \ - ) \ - && tar -zxf ./distribution/target/apache-druid-${VERSION}-bin.tar.gz -C /opt \ - && mv /opt/apache-druid-${VERSION} /opt/druid +COPY ./target/apache-druid-${VERSION}-bin.tar.gz . + +RUN tar -zxf /src/apache-druid-${VERSION}-bin.tar.gz -C /opt \ + && ln -s /opt/apache-druid-${VERSION} /opt/druid FROM alpine:3 as bash-static ARG TARGETARCH @@ -75,6 +59,9 @@ COPY --from=busybox /bin/busybox /busybox/busybox RUN ["/busybox/busybox", "--install", "/bin"] +COPY --from=extractor /opt /opt +COPY ./docker/druid.sh /druid.sh + RUN addgroup -S -g 1000 druid \ && adduser -S -u 1000 -D -H -h /opt/druid -s /bin/sh -g '' -G druid druid diff --git a/distribution/pom.xml b/distribution/pom.xml index ecc00a9155d2..a3c03b0a3104 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -50,6 +50,7 @@ --clean + true @@ -567,5 +568,74 @@ + + docker + + + + com.spotify + dockerfile-maven-plugin + 1.4.10 + + + tag-latest + + build + tag + + + docker.io/apache/incubator-druid + latest + docker/Dockerfile + + + + tag-version + + build + tag + + + docker.io/apache/incubator-druid + ${project.version} + docker/Dockerfile + + + + tag-latest-mysql + + build + tag + + + docker.io/apache/incubator-druid-mysql + latest + docker/Dockerfile.mysql + ${skipDockerMysql} + + + + tag-version-mysql + + build + tag + + + docker.io/apache/incubator-druid-mysql + ${project.version} + docker/Dockerfile.mysql + ${skipDockerMysql} + + + + + + ${project.version} + + + + + + From 160b350b79e85388b4d900d52c90cf3d1167c46f Mon Sep 17 00:00:00 2001 From: Sumit Arrawatia Date: Thu, 16 May 2019 23:23:58 -0700 Subject: [PATCH 002/114] add opencensus extension --- distribution/pom.xml | 1 + .../opencensus-extensions/pom.xml | 132 +++++++++ .../OpenCensusProtobufExtensionsModule.java | 49 ++++ .../OpenCensusProtobufInputRowParser.java | 199 +++++++++++++ ...rg.apache.druid.initialization.DruidModule | 16 ++ .../OpenCensusProtobufInputRowParserTest.java | 267 ++++++++++++++++++ pom.xml | 1 + 7 files changed, 665 insertions(+) create mode 100644 extensions-contrib/opencensus-extensions/pom.xml create mode 100644 extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufExtensionsModule.java create mode 100644 extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java create mode 100755 extensions-contrib/opencensus-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule create mode 100644 extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java diff --git a/distribution/pom.xml b/distribution/pom.xml index a3c03b0a3104..8d417cd6a3b8 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -451,6 +451,7 @@ org.apache.druid.extensions.contrib:opentelemetry-emitter -c org.apache.druid.extensions:druid-iceberg-extensions + org.apache.druid.extensions.contrib:druid-opencensus-extensions diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml new file mode 100644 index 000000000000..33c67ef03392 --- /dev/null +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -0,0 +1,132 @@ + + + + + 4.0.0 + + org.apache.druid.extensions.contrib + druid-opencensus-extensions + druid-opencensus-extensions + druid-opencensus-extensions + + + druid + org.apache.druid + 0.16.0-incubating-SNAPSHOT + ../../pom.xml + + + + 3.2.0 + + + + + io.opencensus + opencensus-proto + 0.2.0 + + + org.apache.druid + druid-core + ${project.parent.version} + provided + + + com.google.protobuf + protobuf-java + ${protobuf.version} + + + com.google.protobuf + protobuf-java-util + ${protobuf.version} + + + com.google.guava + guava + + + + + + junit + junit + test + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.0.0 + + false + + + com.google.protobuf + shaded.com.google.protobuf + + + + + + package + + shade + + + + + + org.apache.maven.plugins + maven-resources-plugin + 3.0.2 + + + desc + + + + + + + + strict + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + -Xep:MissingOverride:WARN + + + + + + + + diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufExtensionsModule.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufExtensionsModule.java new file mode 100644 index 000000000000..39576e4629d0 --- /dev/null +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufExtensionsModule.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opencensus.protobuf; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.inject.Binder; +import org.apache.druid.initialization.DruidModule; + +import java.util.Collections; +import java.util.List; + +public class OpenCensusProtobufExtensionsModule implements DruidModule +{ + + @Override + public List getJacksonModules() + { + return Collections.singletonList( + new SimpleModule("OpenCensusProtobufInputRowParserModule") + .registerSubtypes( + new NamedType(OpenCensusProtobufInputRowParser.class, "opencensus-protobuf") + ) + ); + } + + @Override + public void configure(Binder binder) + { + } +} diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java new file mode 100644 index 000000000000..53b447f1efbe --- /dev/null +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opencensus.protobuf; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import io.opencensus.proto.metrics.v1.Metric; +import io.opencensus.proto.metrics.v1.Point; +import io.opencensus.proto.metrics.v1.TimeSeries; +import org.apache.druid.data.input.ByteBufferInputRowParser; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.MapBasedInputRow; +import org.apache.druid.data.input.impl.ParseSpec; +import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.common.parsers.ParseException; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +public class OpenCensusProtobufInputRowParser implements ByteBufferInputRowParser +{ + private static final Logger LOG = new Logger(OpenCensusProtobufInputRowParser.class); + + private static final String SEPARATOR = "-"; + public static final String NAME = "name"; + public static final String VALUE = "value"; + public static final String TIMESTAMP_COLUMN = "timestamp"; + private final ParseSpec parseSpec; + private final List dimensions; + + @JsonCreator + public OpenCensusProtobufInputRowParser( + @JsonProperty("parseSpec") ParseSpec parseSpec + ) + { + this.parseSpec = parseSpec; + this.dimensions = parseSpec.getDimensionsSpec().getDimensionNames(); + LOG.info("Creating Open Census Protobuf parser with spec:" + parseSpec); + } + + @Override + public ParseSpec getParseSpec() + { + return parseSpec; + } + + @Override + public OpenCensusProtobufInputRowParser withParseSpec(ParseSpec parseSpec) + { + return new OpenCensusProtobufInputRowParser(parseSpec); + } + + @Override + public List parseBatch(ByteBuffer input) + { + + Metric metric; + try { + metric = Metric.parseFrom(ByteString.copyFrom(input)); + } + catch (InvalidProtocolBufferException e) { + throw new ParseException(e, "Protobuf message could not be parsed"); + } + + final List dimensions; + + if (!this.dimensions.isEmpty()) { + dimensions = this.dimensions; + } else { + Set recordDimensions = metric.getMetricDescriptor().getLabelKeysList().stream() + .map(s -> s.getKey()) + .collect(Collectors.toSet()); + recordDimensions.add(NAME); + recordDimensions.add(VALUE); + + + dimensions = Lists.newArrayList( + Sets.difference(recordDimensions, parseSpec.getDimensionsSpec().getDimensionExclusions()) + ); + } + + // Flatten out the OpenCensus record into druid rows. + List rows = new ArrayList<>(); + for (TimeSeries ts : metric.getTimeseriesList()) { + + HashMap labels = new HashMap<>(); + + // Add labels to record. + for (int i = 0; i < metric.getMetricDescriptor().getLabelKeysCount(); i++) { + labels.put(metric.getMetricDescriptor().getLabelKeys(i).getKey(), ts.getLabelValues(i).getValue()); + } + + // One row per timeseries- point. + for (Point point : ts.getPointsList()) { + // Time in millis + labels.put(TIMESTAMP_COLUMN, point.getTimestamp().getSeconds() * 1000); + + switch (point.getValueCase()) { + case DOUBLE_VALUE: + HashMap doubleGauge = new HashMap<>(); + doubleGauge.putAll(labels); + doubleGauge.put(NAME, metric.getMetricDescriptor().getName()); + doubleGauge.put(VALUE, point.getDoubleValue()); + rows.add(new MapBasedInputRow( + parseSpec.getTimestampSpec().extractTimestamp(doubleGauge), + dimensions, + doubleGauge + )); + break; + case INT64_VALUE: + HashMap intGauge = new HashMap<>(); + intGauge.putAll(labels); + intGauge.put(VALUE, point.getInt64Value()); + intGauge.put(NAME, metric.getMetricDescriptor().getName()); + rows.add(new MapBasedInputRow( + parseSpec.getTimestampSpec().extractTimestamp(intGauge), + dimensions, + intGauge + )); + break; + case SUMMARY_VALUE: + // count + HashMap summaryCount = new HashMap<>(); + summaryCount.putAll(labels); + summaryCount.put(NAME, metric.getMetricDescriptor().getName() + SEPARATOR + "count"); + summaryCount.put(VALUE, point.getSummaryValue().getCount().getValue()); + rows.add(new MapBasedInputRow( + parseSpec.getTimestampSpec().extractTimestamp(summaryCount), + dimensions, + summaryCount + )); + + // sum + HashMap summarySum = new HashMap<>(); + summarySum.putAll(labels); + summarySum.put(NAME, metric.getMetricDescriptor().getName() + SEPARATOR + "sum"); + summarySum.put(VALUE, point.getSummaryValue().getSnapshot().getSum().getValue()); + rows.add(new MapBasedInputRow( + parseSpec.getTimestampSpec().extractTimestamp(summarySum), + dimensions, + summarySum + )); + + // TODO : Do we put percentiles into druid ? + break; + case DISTRIBUTION_VALUE: + // count + HashMap distCount = new HashMap<>(); + distCount.put(NAME, metric.getMetricDescriptor().getName() + SEPARATOR + "count"); + distCount.put(VALUE, point.getDistributionValue().getCount()); + rows.add(new MapBasedInputRow( + parseSpec.getTimestampSpec().extractTimestamp(distCount), + dimensions, + distCount + )); + + // sum + HashMap distSum = new HashMap<>(); + distSum.put(NAME, metric.getMetricDescriptor().getName() + SEPARATOR + "sum"); + distSum.put(VALUE, point.getDistributionValue().getSum()); + rows.add(new MapBasedInputRow( + parseSpec.getTimestampSpec().extractTimestamp(distSum), + dimensions, + distSum + )); + // TODO: How to handle buckets ? + break; + } + } + } + return rows; + } + +} diff --git a/extensions-contrib/opencensus-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-contrib/opencensus-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule new file mode 100755 index 000000000000..54b4400fd2cf --- /dev/null +++ b/extensions-contrib/opencensus-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.druid.data.input.opencensus.protobuf.OpenCensusProtobufExtensionsModule \ No newline at end of file diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java new file mode 100644 index 000000000000..8e55755fff5c --- /dev/null +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java @@ -0,0 +1,267 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opencensus.protobuf; + +import com.google.common.collect.Lists; +import com.google.protobuf.DoubleValue; +import com.google.protobuf.Int64Value; +import com.google.protobuf.Timestamp; +import io.opencensus.proto.metrics.v1.LabelKey; +import io.opencensus.proto.metrics.v1.LabelValue; +import io.opencensus.proto.metrics.v1.Metric; +import io.opencensus.proto.metrics.v1.MetricDescriptor; +import io.opencensus.proto.metrics.v1.Point; +import io.opencensus.proto.metrics.v1.SummaryValue; +import io.opencensus.proto.metrics.v1.TimeSeries; +import io.opencensus.proto.resource.v1.Resource; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.JSONParseSpec; +import org.apache.druid.data.input.impl.ParseSpec; +import org.apache.druid.data.input.impl.TimestampSpec; +import org.apache.druid.java.util.common.parsers.JSONPathFieldSpec; +import org.apache.druid.java.util.common.parsers.JSONPathFieldType; +import org.apache.druid.java.util.common.parsers.JSONPathSpec; +import org.joda.time.DateTime; +import org.joda.time.chrono.ISOChronology; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class OpenCensusProtobufInputRowParserTest +{ + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + private ParseSpec parseSpec; + + @Before + public void setUp() + { + parseSpec = new JSONParseSpec( + new TimestampSpec("timestamp", "millis", null), + new DimensionsSpec(null, null, null), + new JSONPathSpec( + true, + Lists.newArrayList( + new JSONPathFieldSpec(JSONPathFieldType.ROOT, "name", ""), + new JSONPathFieldSpec(JSONPathFieldType.ROOT, "value", ""), + new JSONPathFieldSpec(JSONPathFieldType.ROOT, "foo_key", "") + ) + ), null + ); + + } + + + @Test + public void testGaugeParse() throws Exception + { + + //configure parser with desc file + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec); + + DateTime dateTime = new DateTime(2019, 07, 12, 9, 30, ISOChronology.getInstanceUTC()); + + Timestamp timestamp = Timestamp.newBuilder().setSeconds(dateTime.getMillis() / 1000) + .setNanos((int) ((dateTime.getMillis() % 1000) * 1000000)).build(); + + System.out.println(timestamp.getSeconds() * 1000); + + Metric d = doubleGaugeMetric(timestamp); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + d.writeTo(out); + + InputRow row = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())).get(0); + assertEquals(dateTime.getMillis(), row.getTimestampFromEpoch()); + + assertDimensionEquals(row, "name", "metric_gauge_double"); + assertDimensionEquals(row, "foo_key", "foo_value"); + + + assertEquals(2000, row.getMetric("value").doubleValue(), 0.0); + } + + @Test + public void testSummaryParse() throws Exception + { + //configure parser with desc file + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec); + + DateTime dateTime = new DateTime(2019, 07, 12, 9, 30, ISOChronology.getInstanceUTC()); + + Timestamp timestamp = Timestamp.newBuilder().setSeconds(dateTime.getMillis() / 1000) + .setNanos((int) ((dateTime.getMillis() % 1000) * 1000000)).build(); + + Metric d = summaryMetric(timestamp); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + d.writeTo(out); + + List rows = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())); + + assertEquals(2, rows.size()); + + InputRow row = rows.get(0); + assertEquals(dateTime.getMillis(), row.getTimestampFromEpoch()); + assertDimensionEquals(row, "name", "metric_summary-count"); + assertDimensionEquals(row, "foo_key", "foo_value"); + assertEquals(40, row.getMetric("value").doubleValue(), 0.0); + + row = rows.get(1); + assertEquals(dateTime.getMillis(), row.getTimestampFromEpoch()); + assertDimensionEquals(row, "name", "metric_summary-sum"); + assertDimensionEquals(row, "foo_key", "foo_value"); + assertEquals(10, row.getMetric("value").doubleValue(), 0.0); + + } + + private void assertDimensionEquals(InputRow row, String dimension, Object expected) + { + List values = row.getDimension(dimension); + + assertEquals(1, values.size()); + assertEquals(expected, values.get(0)); + } + + private Metric doubleGaugeMetric(Timestamp timestamp) + { + Metric dist = Metric.newBuilder() + .setMetricDescriptor( + MetricDescriptor.newBuilder() + .setName("metric_gauge_double") + .setDescription("metric_gauge_double_description") + .setUnit("ms") + .setType( + MetricDescriptor.Type.GAUGE_DOUBLE) + .addLabelKeys( + LabelKey.newBuilder() + .setKey("foo_key") + .build()) + .build()) + .setResource( + Resource.newBuilder() + .setType("env") + .putAllLabels(Collections.singletonMap("env_key", "env_val")) + .build()) + .addTimeseries( + TimeSeries.newBuilder() + .setStartTimestamp(timestamp) + .addLabelValues( + LabelValue.newBuilder() + .setHasValue(true) + .setValue("foo_value") + .build()) + .addPoints( + Point.newBuilder() + .setTimestamp(timestamp) + .setDoubleValue(2000) + .build()) + .build()) + .build(); + + return dist; + } + + + private Metric summaryMetric(Timestamp timestamp) + { + + SummaryValue.Snapshot snapshot = SummaryValue.Snapshot.newBuilder() + .setSum(DoubleValue.newBuilder().setValue(10).build()) + .addPercentileValues(SummaryValue.Snapshot.ValueAtPercentile.newBuilder() + .setPercentile(50.0) + .setValue(10) + .build()) + .addPercentileValues(SummaryValue.Snapshot.ValueAtPercentile.newBuilder() + .setPercentile(75.0) + .setValue(20) + .build()) + .addPercentileValues(SummaryValue.Snapshot.ValueAtPercentile.newBuilder() + .setPercentile(95.0) + .setValue(30) + .build()) + .addPercentileValues(SummaryValue.Snapshot.ValueAtPercentile.newBuilder() + .setPercentile(98.0) + .setValue(40) + .build()) + .addPercentileValues(SummaryValue.Snapshot.ValueAtPercentile.newBuilder() + .setPercentile(99.0) + .setValue(50) + .build()) + .addPercentileValues(SummaryValue.Snapshot.ValueAtPercentile.newBuilder() + .setPercentile(99.9) + .setValue(60) + .build()) + .build(); + + + SummaryValue summaryValue = SummaryValue.newBuilder() + .setCount(Int64Value.newBuilder().setValue(40).build()) + .setSnapshot(snapshot) + .build(); + + + Metric dist = Metric.newBuilder() + .setMetricDescriptor( + MetricDescriptor.newBuilder() + .setName("metric_summary") + .setDescription("metric_summary_description") + .setUnit("ms") + .setType( + MetricDescriptor.Type.SUMMARY) + .addLabelKeys( + LabelKey.newBuilder() + .setKey("foo_key") + .build()) + .build()) + .setResource( + Resource.newBuilder() + .setType("env") + .putAllLabels(Collections.singletonMap("env_key", "env_val")) + .build()) + .addTimeseries( + TimeSeries.newBuilder() + .setStartTimestamp(timestamp) + .addLabelValues( + LabelValue.newBuilder() + .setHasValue(true) + .setValue("foo_value") + .build()) + .addPoints( + Point.newBuilder() + .setTimestamp(timestamp) + .setSummaryValue(summaryValue) + .build()) + .build()) + .build(); + + return dist; + } + + +} diff --git a/pom.xml b/pom.xml index 5fb1f49b2f2e..6fc529563b1e 100644 --- a/pom.xml +++ b/pom.xml @@ -227,6 +227,7 @@ extensions-contrib/opentelemetry-emitter extensions-contrib/kubernetes-overlord-extensions extensions-contrib/druid-iceberg-extensions + extensions-contrib/opencensus-extensions distribution From fc03683d831845df28fbcf6225c44afc5e8acefa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Fri, 26 Jul 2019 12:20:23 -0700 Subject: [PATCH 003/114] make checkstyle happy --- .../OpenCensusProtobufInputRowParserTest.java | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java index 8e55755fff5c..f664ab5527a1 100644 --- a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java @@ -41,6 +41,7 @@ import org.apache.druid.java.util.common.parsers.JSONPathSpec; import org.joda.time.DateTime; import org.joda.time.chrono.ISOChronology; +import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -51,8 +52,6 @@ import java.util.Collections; import java.util.List; -import static org.junit.Assert.assertEquals; - public class OpenCensusProtobufInputRowParserTest { @Rule @@ -98,13 +97,13 @@ public void testGaugeParse() throws Exception d.writeTo(out); InputRow row = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())).get(0); - assertEquals(dateTime.getMillis(), row.getTimestampFromEpoch()); + Assert.assertEquals(dateTime.getMillis(), row.getTimestampFromEpoch()); assertDimensionEquals(row, "name", "metric_gauge_double"); assertDimensionEquals(row, "foo_key", "foo_value"); - assertEquals(2000, row.getMetric("value").doubleValue(), 0.0); + Assert.assertEquals(2000, row.getMetric("value").doubleValue(), 0.0); } @Test @@ -124,19 +123,19 @@ public void testSummaryParse() throws Exception List rows = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())); - assertEquals(2, rows.size()); + Assert.assertEquals(2, rows.size()); InputRow row = rows.get(0); - assertEquals(dateTime.getMillis(), row.getTimestampFromEpoch()); + Assert.assertEquals(dateTime.getMillis(), row.getTimestampFromEpoch()); assertDimensionEquals(row, "name", "metric_summary-count"); assertDimensionEquals(row, "foo_key", "foo_value"); - assertEquals(40, row.getMetric("value").doubleValue(), 0.0); + Assert.assertEquals(40, row.getMetric("value").doubleValue(), 0.0); row = rows.get(1); - assertEquals(dateTime.getMillis(), row.getTimestampFromEpoch()); + Assert.assertEquals(dateTime.getMillis(), row.getTimestampFromEpoch()); assertDimensionEquals(row, "name", "metric_summary-sum"); assertDimensionEquals(row, "foo_key", "foo_value"); - assertEquals(10, row.getMetric("value").doubleValue(), 0.0); + Assert.assertEquals(10, row.getMetric("value").doubleValue(), 0.0); } @@ -144,8 +143,8 @@ private void assertDimensionEquals(InputRow row, String dimension, Object expect { List values = row.getDimension(dimension); - assertEquals(1, values.size()); - assertEquals(expected, values.get(0)); + Assert.assertEquals(1, values.size()); + Assert.assertEquals(expected, values.get(0)); } private Metric doubleGaugeMetric(Timestamp timestamp) From 40a452551b3c365f20726a22bed431921c8b9711 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Wed, 4 Sep 2019 23:47:58 -0700 Subject: [PATCH 004/114] bump pom version for opencensus extension --- extensions-contrib/opencensus-extensions/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml index 33c67ef03392..52607088a0be 100644 --- a/extensions-contrib/opencensus-extensions/pom.xml +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -31,7 +31,7 @@ druid org.apache.druid - 0.16.0-incubating-SNAPSHOT + 0.17.0-incubating-SNAPSHOT ../../pom.xml From f497af42a670e8d2491c9b5f738418914a8cdc25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 10 Dec 2019 09:36:18 -0800 Subject: [PATCH 005/114] fix issues related to shading opencensus extension The extension packaging included both shaded and unshaded dependencies in the classpath. Shading should not be necessary in this case. Also excludes guava dependencies, which are already provided by Druid and don't need to be added to the extensions jars. --- .../opencensus-extensions/pom.xml | 65 +++---------------- 1 file changed, 8 insertions(+), 57 deletions(-) diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml index 52607088a0be..3c7fd8c9c026 100644 --- a/extensions-contrib/opencensus-extensions/pom.xml +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -44,29 +44,21 @@ io.opencensus opencensus-proto 0.2.0 - - - org.apache.druid - druid-core - ${project.parent.version} - provided - - - com.google.protobuf - protobuf-java - ${protobuf.version} - - - com.google.protobuf - protobuf-java-util - ${protobuf.version} + com.google.guava guava + + org.apache.druid + druid-core + ${project.parent.version} + provided + junit @@ -76,28 +68,6 @@ - - org.apache.maven.plugins - maven-shade-plugin - 3.0.0 - - false - - - com.google.protobuf - shaded.com.google.protobuf - - - - - - package - - shade - - - - org.apache.maven.plugins maven-resources-plugin @@ -110,23 +80,4 @@ - - - strict - - - - org.apache.maven.plugins - maven-compiler-plugin - - - - -Xep:MissingOverride:WARN - - - - - - - From 1a07ece303618ab027ebc3c73879768e0133f055 Mon Sep 17 00:00:00 2001 From: Apoorv Mittal Date: Fri, 14 Feb 2020 18:14:08 +0530 Subject: [PATCH 006/114] METRICS-516: Adding Resource labels in OpenCensus Extension --- .../OpenCensusProtobufInputRowParser.java | 71 +++++++-------- .../OpenCensusProtobufInputRowParserTest.java | 90 ++++++++++++++++++- 2 files changed, 119 insertions(+), 42 deletions(-) diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java index 53b447f1efbe..252cac590525 100644 --- a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java @@ -39,6 +39,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -95,10 +96,16 @@ public List parseBatch(ByteBuffer input) Set recordDimensions = metric.getMetricDescriptor().getLabelKeysList().stream() .map(s -> s.getKey()) .collect(Collectors.toSet()); + + // Add resource map key set to record dimensions. + recordDimensions.addAll(metric.getResource().getLabelsMap().keySet()); + + // NAME, VALUE dimensions will not be present in labelKeysList or Metric.Resource map as they + // are derived dimensions, which get populated while parsing data for timeSeries hence add + // them to recordDimensions. recordDimensions.add(NAME); recordDimensions.add(VALUE); - dimensions = Lists.newArrayList( Sets.difference(recordDimensions, parseSpec.getDimensionsSpec().getDimensionExclusions()) ); @@ -108,92 +115,80 @@ public List parseBatch(ByteBuffer input) List rows = new ArrayList<>(); for (TimeSeries ts : metric.getTimeseriesList()) { - HashMap labels = new HashMap<>(); + // Add common resourceLabels. + Map labels = new HashMap<>(metric.getResource().getLabelsMap()); // Add labels to record. for (int i = 0; i < metric.getMetricDescriptor().getLabelKeysCount(); i++) { labels.put(metric.getMetricDescriptor().getLabelKeys(i).getKey(), ts.getLabelValues(i).getValue()); } - // One row per timeseries- point. + // One row per timeSeries point. for (Point point : ts.getPointsList()) { // Time in millis labels.put(TIMESTAMP_COLUMN, point.getTimestamp().getSeconds() * 1000); switch (point.getValueCase()) { case DOUBLE_VALUE: - HashMap doubleGauge = new HashMap<>(); + Map doubleGauge = new HashMap<>(); doubleGauge.putAll(labels); doubleGauge.put(NAME, metric.getMetricDescriptor().getName()); doubleGauge.put(VALUE, point.getDoubleValue()); - rows.add(new MapBasedInputRow( - parseSpec.getTimestampSpec().extractTimestamp(doubleGauge), - dimensions, - doubleGauge - )); + addDerivedMetricsRow(doubleGauge, dimensions, rows); break; case INT64_VALUE: HashMap intGauge = new HashMap<>(); intGauge.putAll(labels); intGauge.put(VALUE, point.getInt64Value()); intGauge.put(NAME, metric.getMetricDescriptor().getName()); - rows.add(new MapBasedInputRow( - parseSpec.getTimestampSpec().extractTimestamp(intGauge), - dimensions, - intGauge - )); + addDerivedMetricsRow(intGauge, dimensions, rows); break; case SUMMARY_VALUE: // count - HashMap summaryCount = new HashMap<>(); + Map summaryCount = new HashMap<>(); summaryCount.putAll(labels); summaryCount.put(NAME, metric.getMetricDescriptor().getName() + SEPARATOR + "count"); summaryCount.put(VALUE, point.getSummaryValue().getCount().getValue()); - rows.add(new MapBasedInputRow( - parseSpec.getTimestampSpec().extractTimestamp(summaryCount), - dimensions, - summaryCount - )); + addDerivedMetricsRow(summaryCount, dimensions, rows); // sum - HashMap summarySum = new HashMap<>(); + Map summarySum = new HashMap<>(); summarySum.putAll(labels); summarySum.put(NAME, metric.getMetricDescriptor().getName() + SEPARATOR + "sum"); summarySum.put(VALUE, point.getSummaryValue().getSnapshot().getSum().getValue()); - rows.add(new MapBasedInputRow( - parseSpec.getTimestampSpec().extractTimestamp(summarySum), - dimensions, - summarySum - )); + addDerivedMetricsRow(summarySum, dimensions, rows); // TODO : Do we put percentiles into druid ? break; case DISTRIBUTION_VALUE: // count - HashMap distCount = new HashMap<>(); + Map distCount = new HashMap<>(); distCount.put(NAME, metric.getMetricDescriptor().getName() + SEPARATOR + "count"); distCount.put(VALUE, point.getDistributionValue().getCount()); - rows.add(new MapBasedInputRow( - parseSpec.getTimestampSpec().extractTimestamp(distCount), - dimensions, - distCount - )); + addDerivedMetricsRow(distCount, dimensions, rows); // sum - HashMap distSum = new HashMap<>(); + Map distSum = new HashMap<>(); distSum.put(NAME, metric.getMetricDescriptor().getName() + SEPARATOR + "sum"); distSum.put(VALUE, point.getDistributionValue().getSum()); - rows.add(new MapBasedInputRow( - parseSpec.getTimestampSpec().extractTimestamp(distSum), - dimensions, - distSum - )); + addDerivedMetricsRow(distSum, dimensions, rows); // TODO: How to handle buckets ? break; } } } + return rows; } + private void addDerivedMetricsRow(Map derivedMetrics, List dimensions, + List rows) + { + rows.add(new MapBasedInputRow( + parseSpec.getTimestampSpec().extractTimestamp(derivedMetrics), + dimensions, + derivedMetrics + )); + } + } diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java index f664ab5527a1..88a422585495 100644 --- a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java @@ -19,6 +19,7 @@ package org.apache.druid.data.input.opencensus.protobuf; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.protobuf.DoubleValue; import com.google.protobuf.Int64Value; @@ -35,6 +36,7 @@ import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.JSONParseSpec; import org.apache.druid.data.input.impl.ParseSpec; +import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.parsers.JSONPathFieldSpec; import org.apache.druid.java.util.common.parsers.JSONPathFieldType; @@ -59,6 +61,8 @@ public class OpenCensusProtobufInputRowParserTest private ParseSpec parseSpec; + private ParseSpec parseSpecWithDimensions; + @Before public void setUp() { @@ -75,6 +79,20 @@ public void setUp() ), null ); + parseSpecWithDimensions = new JSONParseSpec( + new TimestampSpec("timestamp", "millis", null), + new DimensionsSpec(ImmutableList.of( + new StringDimensionSchema("foo_key"), + new StringDimensionSchema("env_key")), null, null), + new JSONPathSpec( + true, + Lists.newArrayList( + new JSONPathFieldSpec(JSONPathFieldType.ROOT, "name", ""), + new JSONPathFieldSpec(JSONPathFieldType.ROOT, "value", ""), + new JSONPathFieldSpec(JSONPathFieldType.ROOT, "foo_key", "") + ) + ), null + ); } @@ -92,9 +110,9 @@ public void testGaugeParse() throws Exception System.out.println(timestamp.getSeconds() * 1000); - Metric d = doubleGaugeMetric(timestamp); + Metric metric = doubleGaugeMetric(timestamp); ByteArrayOutputStream out = new ByteArrayOutputStream(); - d.writeTo(out); + metric.writeTo(out); InputRow row = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())).get(0); Assert.assertEquals(dateTime.getMillis(), row.getTimestampFromEpoch()); @@ -117,9 +135,9 @@ public void testSummaryParse() throws Exception Timestamp timestamp = Timestamp.newBuilder().setSeconds(dateTime.getMillis() / 1000) .setNanos((int) ((dateTime.getMillis() % 1000) * 1000000)).build(); - Metric d = summaryMetric(timestamp); + Metric metric = summaryMetric(timestamp); ByteArrayOutputStream out = new ByteArrayOutputStream(); - d.writeTo(out); + metric.writeTo(out); List rows = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())); @@ -139,6 +157,70 @@ public void testSummaryParse() throws Exception } + @Test + public void testDimensionsParseWithParseSpecDimensions() throws Exception + { + //configure parser with desc file + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpecWithDimensions); + + DateTime dateTime = new DateTime(2019, 07, 12, 9, 30, ISOChronology.getInstanceUTC()); + + Timestamp timestamp = Timestamp.newBuilder().setSeconds(dateTime.getMillis() / 1000) + .setNanos((int) ((dateTime.getMillis() % 1000) * 1000000)).build(); + + Metric metric = summaryMetric(timestamp); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + metric.writeTo(out); + + List rows = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())); + + Assert.assertEquals(2, rows.size()); + + InputRow row = rows.get(0); + Assert.assertEquals(2, row.getDimensions().size()); + assertDimensionEquals(row, "env_key", "env_val"); + assertDimensionEquals(row, "foo_key", "foo_value"); + + row = rows.get(1); + Assert.assertEquals(2, row.getDimensions().size()); + assertDimensionEquals(row, "env_key", "env_val"); + assertDimensionEquals(row, "foo_key", "foo_value"); + + } + + @Test + public void testDimensionsParseWithoutParseSpecDimensions() throws Exception + { + //configure parser with desc file + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec); + + DateTime dateTime = new DateTime(2019, 07, 12, 9, 30, ISOChronology.getInstanceUTC()); + + Timestamp timestamp = Timestamp.newBuilder().setSeconds(dateTime.getMillis() / 1000) + .setNanos((int) ((dateTime.getMillis() % 1000) * 1000000)).build(); + + Metric metric = summaryMetric(timestamp); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + metric.writeTo(out); + + List rows = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())); + + Assert.assertEquals(2, rows.size()); + + InputRow row = rows.get(0); + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "name", "metric_summary-count"); + assertDimensionEquals(row, "env_key", "env_val"); + assertDimensionEquals(row, "foo_key", "foo_value"); + + row = rows.get(1); + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "name", "metric_summary-sum"); + assertDimensionEquals(row, "env_key", "env_val"); + assertDimensionEquals(row, "foo_key", "foo_value"); + + } + private void assertDimensionEquals(InputRow row, String dimension, Object expected) { List values = row.getDimension(dimension); From bb77616fbd74357a059eabdd05d6db0812f73554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Fri, 13 Mar 2020 14:20:03 -0700 Subject: [PATCH 007/114] bump extension version to match release --- extensions-contrib/opencensus-extensions/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml index 3c7fd8c9c026..503f81925f80 100644 --- a/extensions-contrib/opencensus-extensions/pom.xml +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -31,7 +31,7 @@ druid org.apache.druid - 0.17.0-incubating-SNAPSHOT + 0.17.0 ../../pom.xml From 226f2b697c820412c9b5357427f93b015154dac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 24 Mar 2020 11:06:18 -0700 Subject: [PATCH 008/114] confluent-extensions with custom transform specs (#9) --- codestyle/checkstyle-suppressions.xml | 4 + distribution/pom.xml | 2 + .../confluent-extensions/pom.xml | 44 ++++++ .../druid/ConfluentExtensionsModule.java | 36 +++++ .../ExtractTenantTopicTransform.java | 45 ++++++ .../transform/ExtractTenantTransform.java | 45 ++++++ .../druid/transform/TenantUtils.java | 26 ++++ ...rg.apache.druid.initialization.DruidModule | 3 + .../druid/transform/ExtractTransformTest.java | 133 ++++++++++++++++++ pom.xml | 1 + 10 files changed, 339 insertions(+) create mode 100644 extensions-contrib/confluent-extensions/pom.xml create mode 100644 extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/ConfluentExtensionsModule.java create mode 100644 extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTopicTransform.java create mode 100644 extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTransform.java create mode 100644 extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/TenantUtils.java create mode 100644 extensions-contrib/confluent-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule create mode 100644 extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java diff --git a/codestyle/checkstyle-suppressions.xml b/codestyle/checkstyle-suppressions.xml index 01868d73f34f..6d42d891382f 100644 --- a/codestyle/checkstyle-suppressions.xml +++ b/codestyle/checkstyle-suppressions.xml @@ -70,4 +70,8 @@ + + + + diff --git a/distribution/pom.xml b/distribution/pom.xml index 8d417cd6a3b8..e40ca8c5c4c2 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -452,6 +452,8 @@ -c org.apache.druid.extensions:druid-iceberg-extensions org.apache.druid.extensions.contrib:druid-opencensus-extensions + -c + io.confluent.druid.extensions:confluent-extensions diff --git a/extensions-contrib/confluent-extensions/pom.xml b/extensions-contrib/confluent-extensions/pom.xml new file mode 100644 index 000000000000..33c0f29c1914 --- /dev/null +++ b/extensions-contrib/confluent-extensions/pom.xml @@ -0,0 +1,44 @@ + + + + + + 4.0.0 + + io.confluent.druid.extensions + confluent-extensions + confluent-extensions + confluent-extensions + + + druid + org.apache.druid + 0.17.0 + ../../pom.xml + + + + + org.apache.druid + druid-core + ${project.parent.version} + provided + + + org.apache.druid + druid-processing + ${project.parent.version} + provided + + + + junit + junit + test + + + diff --git a/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/ConfluentExtensionsModule.java b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/ConfluentExtensionsModule.java new file mode 100644 index 000000000000..a2a835a10cde --- /dev/null +++ b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/ConfluentExtensionsModule.java @@ -0,0 +1,36 @@ +/* + * Copyright 2020 Confluent Inc. + */ + +package io.confluent.druid; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.inject.Binder; +import io.confluent.druid.transform.ExtractTenantTopicTransform; +import io.confluent.druid.transform.ExtractTenantTransform; +import org.apache.druid.initialization.DruidModule; + +import java.util.Collections; +import java.util.List; + +public class ConfluentExtensionsModule implements DruidModule +{ + @Override + public List getJacksonModules() + { + return Collections.singletonList( + new SimpleModule("ConfluentTransformsModule") + .registerSubtypes( + new NamedType(ExtractTenantTransform.class, "extractTenant"), + new NamedType(ExtractTenantTopicTransform.class, "extractTenantTopic") + ) + ); + } + + @Override + public void configure(Binder binder) + { + } +} diff --git a/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTopicTransform.java b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTopicTransform.java new file mode 100644 index 000000000000..0c1ed81fdf0d --- /dev/null +++ b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTopicTransform.java @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Confluent Inc. + */ + +package io.confluent.druid.transform; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.segment.transform.RowFunction; +import org.apache.druid.segment.transform.Transform; + +public class ExtractTenantTopicTransform implements Transform +{ + private final String fieldName; + private final String name; + + public ExtractTenantTopicTransform( + @JsonProperty("name") final String name, + @JsonProperty("fieldName") final String fieldName + ) + { + this.fieldName = fieldName; + this.name = name; + } + + @Override + public String getName() + { + return name; + } + + @Override + public RowFunction getRowFunction() + { + return row -> { + Object existing = row.getRaw(name); + // do not overwrite existing values if present + if (existing != null) { + return existing; + } + + Object value = row.getRaw(fieldName); + return value == null ? null : TenantUtils.extractTenantTopic(value.toString()); + }; + } +} diff --git a/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTransform.java b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTransform.java new file mode 100644 index 000000000000..a9938b01e3c5 --- /dev/null +++ b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTransform.java @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Confluent Inc. + */ + +package io.confluent.druid.transform; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.segment.transform.RowFunction; +import org.apache.druid.segment.transform.Transform; + +public class ExtractTenantTransform implements Transform +{ + private final String fieldName; + private final String name; + + public ExtractTenantTransform( + @JsonProperty("name") final String name, + @JsonProperty("fieldName") final String fieldName + ) + { + this.fieldName = fieldName; + this.name = name; + } + + @Override + public String getName() + { + return name; + } + + @Override + public RowFunction getRowFunction() + { + return row -> { + Object existing = row.getRaw(name); + // do not overwrite existing values if present + if (existing != null) { + return existing; + } + + Object value = row.getRaw(fieldName); + return value == null ? null : TenantUtils.extractTenant(value.toString()); + }; + } +} diff --git a/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/TenantUtils.java b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/TenantUtils.java new file mode 100644 index 000000000000..3de3bcba641d --- /dev/null +++ b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/TenantUtils.java @@ -0,0 +1,26 @@ +/* + * Copyright 2020 Confluent Inc. + */ + +package io.confluent.druid.transform; + +import javax.annotation.Nullable; + +public class TenantUtils +{ + private static final char DELIMITER = '_'; + + @Nullable + public static String extractTenant(String prefixedTopic) + { + int i = prefixedTopic.indexOf(DELIMITER); + return i < 0 ? null : prefixedTopic.substring(0, i); + } + + @Nullable + public static String extractTenantTopic(String prefixedTopic) + { + int i = prefixedTopic.indexOf(DELIMITER); + return i < 0 ? null : prefixedTopic.substring(i + 1); + } +} diff --git a/extensions-contrib/confluent-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-contrib/confluent-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule new file mode 100644 index 000000000000..f14e0fe0915b --- /dev/null +++ b/extensions-contrib/confluent-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -0,0 +1,3 @@ +# Copyright 2020 Confluent Inc. + +io.confluent.druid.ConfluentExtensionsModule diff --git a/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java b/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java new file mode 100644 index 000000000000..2aeab583a393 --- /dev/null +++ b/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java @@ -0,0 +1,133 @@ +/* + * Copyright 2020 Confluent Inc. + */ + +package io.confluent.druid.transform; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.InputRowParser; +import org.apache.druid.data.input.impl.MapInputRowParser; +import org.apache.druid.data.input.impl.TimeAndDimsParseSpec; +import org.apache.druid.data.input.impl.TimestampSpec; +import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.segment.transform.TransformSpec; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Map; + +public class ExtractTransformTest +{ + + private static final MapInputRowParser PARSER = new MapInputRowParser( + new TimeAndDimsParseSpec( + new TimestampSpec("t", "auto", DateTimes.of("2020-01-01")), + new DimensionsSpec( + DimensionsSpec.getDefaultSchemas(ImmutableList.of("topic", "tenant")), + null, + null + ) + ) + ); + + private static final Map ROW1 = ImmutableMap.builder() + .put("topic", "lkc-abc123_mytopic") + .build(); + + private static final Map ROW2 = ImmutableMap.builder() + .put("tenant", "lkc-xyz789") + .put("tenant_topic", "topic0") + .put("topic", "lkc-abc123_mytopic") + .build(); + + private static final Map ROW3 = ImmutableMap.builder() + .put("topic", "invalid-topic") + .build(); + + private static final Map ROW4 = ImmutableMap.builder() + .build(); + + + @Test + public void testExtraction() + { + final TransformSpec transformSpec = new TransformSpec( + null, + ImmutableList.of( + new ExtractTenantTransform("tenant", "topic"), + new ExtractTenantTopicTransform("tenant_topic", "topic") + ) + ); + + final InputRowParser> parser = transformSpec.decorate(PARSER); + final InputRow row = parser.parseBatch(ROW1).get(0); + + Assert.assertNotNull(row); + Assert.assertEquals(ImmutableList.of("topic", "tenant"), row.getDimensions()); + Assert.assertEquals(ImmutableList.of("lkc-abc123"), row.getDimension("tenant")); + Assert.assertEquals(ImmutableList.of("mytopic"), row.getDimension("tenant_topic")); + } + + @Test + public void testPreserveExistingFields() + { + final TransformSpec transformSpec = new TransformSpec( + null, + ImmutableList.of( + new ExtractTenantTransform("tenant", "topic"), + new ExtractTenantTopicTransform("tenant_topic", "topic") + ) + ); + + final InputRowParser> parser = transformSpec.decorate(PARSER); + final InputRow row = parser.parseBatch(ROW2).get(0); + + Assert.assertNotNull(row); + Assert.assertEquals(ImmutableList.of("topic", "tenant"), row.getDimensions()); + Assert.assertEquals(ImmutableList.of("lkc-xyz789"), row.getDimension("tenant")); + Assert.assertEquals(ImmutableList.of("topic0"), row.getDimension("tenant_topic")); + } + + @Test + public void testInvalidTopics() + { + final TransformSpec transformSpec = new TransformSpec( + null, + ImmutableList.of( + new ExtractTenantTransform("tenant", "topic"), + new ExtractTenantTopicTransform("tenant_topic", "topic") + ) + ); + + final InputRowParser> parser = transformSpec.decorate(PARSER); + final InputRow row = parser.parseBatch(ROW3).get(0); + + Assert.assertNotNull(row); + Assert.assertEquals(ImmutableList.of("topic", "tenant"), row.getDimensions()); + Assert.assertNull(row.getRaw("tenant")); + Assert.assertNull(row.getRaw("tenant_topic")); + } + + @Test + public void testNullTopic() + { + final TransformSpec transformSpec = new TransformSpec( + null, + ImmutableList.of( + new ExtractTenantTransform("tenant", "topic"), + new ExtractTenantTopicTransform("tenant_topic", "topic") + ) + ); + + final InputRowParser> parser = transformSpec.decorate(PARSER); + final InputRow row = parser.parseBatch(ROW4).get(0); + + Assert.assertNotNull(row); + Assert.assertEquals(ImmutableList.of("topic", "tenant"), row.getDimensions()); + Assert.assertNull(row.getRaw("tenant")); + Assert.assertNull(row.getRaw("tenant_topic")); + } +} diff --git a/pom.xml b/pom.xml index 6fc529563b1e..fdc4bb07a41f 100644 --- a/pom.xml +++ b/pom.xml @@ -228,6 +228,7 @@ extensions-contrib/kubernetes-overlord-extensions extensions-contrib/druid-iceberg-extensions extensions-contrib/opencensus-extensions + extensions-contrib/confluent-extensions distribution From 8db469ef56c4a57b706a36284862ecb98acc13b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Thu, 26 Mar 2020 09:19:37 -0700 Subject: [PATCH 009/114] fix extraction transform serde (#10) --- .../confluent-extensions/pom.xml | 7 +++ .../ExtractTenantTopicTransform.java | 43 ++++++++++++++++++- .../transform/ExtractTenantTransform.java | 43 ++++++++++++++++++- .../druid/transform/ExtractTransformTest.java | 24 +++++++++++ 4 files changed, 113 insertions(+), 4 deletions(-) diff --git a/extensions-contrib/confluent-extensions/pom.xml b/extensions-contrib/confluent-extensions/pom.xml index 33c0f29c1914..0c640d272185 100644 --- a/extensions-contrib/confluent-extensions/pom.xml +++ b/extensions-contrib/confluent-extensions/pom.xml @@ -40,5 +40,12 @@ junit test + + org.apache.druid + druid-processing + ${project.parent.version} + test + test-jar + diff --git a/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTopicTransform.java b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTopicTransform.java index 0c1ed81fdf0d..6dcad8460502 100644 --- a/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTopicTransform.java +++ b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTopicTransform.java @@ -5,9 +5,12 @@ package io.confluent.druid.transform; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; import org.apache.druid.segment.transform.RowFunction; import org.apache.druid.segment.transform.Transform; +import java.util.Objects; + public class ExtractTenantTopicTransform implements Transform { private final String fieldName; @@ -18,16 +21,23 @@ public ExtractTenantTopicTransform( @JsonProperty("fieldName") final String fieldName ) { - this.fieldName = fieldName; - this.name = name; + this.name = Preconditions.checkNotNull(name, "name"); + this.fieldName = Preconditions.checkNotNull(fieldName, "fieldName"); } + @JsonProperty @Override public String getName() { return name; } + @JsonProperty + public String getFieldName() + { + return fieldName; + } + @Override public RowFunction getRowFunction() { @@ -42,4 +52,33 @@ public RowFunction getRowFunction() return value == null ? null : TenantUtils.extractTenantTopic(value.toString()); }; } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (!(o instanceof ExtractTenantTopicTransform)) { + return false; + } + ExtractTenantTopicTransform that = (ExtractTenantTopicTransform) o; + return fieldName.equals(that.fieldName) && + name.equals(that.name); + } + + @Override + public int hashCode() + { + return Objects.hash(fieldName, name); + } + + @Override + public String toString() + { + return "ExtractTenantTopicTransform{" + + "fieldName='" + fieldName + '\'' + + ", name='" + name + '\'' + + '}'; + } } diff --git a/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTransform.java b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTransform.java index a9938b01e3c5..b4b392146ae9 100644 --- a/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTransform.java +++ b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTransform.java @@ -5,9 +5,12 @@ package io.confluent.druid.transform; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; import org.apache.druid.segment.transform.RowFunction; import org.apache.druid.segment.transform.Transform; +import java.util.Objects; + public class ExtractTenantTransform implements Transform { private final String fieldName; @@ -18,16 +21,23 @@ public ExtractTenantTransform( @JsonProperty("fieldName") final String fieldName ) { - this.fieldName = fieldName; - this.name = name; + this.name = Preconditions.checkNotNull(name, "name"); + this.fieldName = Preconditions.checkNotNull(fieldName, "fieldName"); } + @JsonProperty @Override public String getName() { return name; } + @JsonProperty + public String getFieldName() + { + return fieldName; + } + @Override public RowFunction getRowFunction() { @@ -42,4 +52,33 @@ public RowFunction getRowFunction() return value == null ? null : TenantUtils.extractTenant(value.toString()); }; } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (!(o instanceof ExtractTenantTransform)) { + return false; + } + ExtractTenantTransform that = (ExtractTenantTransform) o; + return fieldName.equals(that.fieldName) && + name.equals(that.name); + } + + @Override + public int hashCode() + { + return Objects.hash(fieldName, name); + } + + @Override + public String toString() + { + return "ExtractTenantTransform{" + + "fieldName='" + fieldName + '\'' + + ", name='" + name + '\'' + + '}'; + } } diff --git a/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java b/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java index 2aeab583a393..8c1d60c5fc99 100644 --- a/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java +++ b/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java @@ -4,8 +4,11 @@ package io.confluent.druid.transform; +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.confluent.druid.ConfluentExtensionsModule; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.InputRowParser; @@ -13,6 +16,7 @@ import org.apache.druid.data.input.impl.TimeAndDimsParseSpec; import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.DateTimes; +import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.transform.TransformSpec; import org.junit.Assert; import org.junit.Test; @@ -130,4 +134,24 @@ public void testNullTopic() Assert.assertNull(row.getRaw("tenant")); Assert.assertNull(row.getRaw("tenant_topic")); } + + @Test + public void testSerde() throws Exception + { + final TransformSpec transformSpec = new TransformSpec( + null, + ImmutableList.of( + new ExtractTenantTopicTransform("tenant_topic", "topic"), + new ExtractTenantTransform("tenant", "topic") + ) + ); + + final ObjectMapper jsonMapper = TestHelper.makeJsonMapper(); + jsonMapper.registerModules(new ConfluentExtensionsModule().getJacksonModules()); + + Assert.assertEquals( + transformSpec, + jsonMapper.readValue(jsonMapper.writeValueAsString(transformSpec), TransformSpec.class) + ); + } } From fe3a1f11aba43c83db3b7eb2661d31397305b1c2 Mon Sep 17 00:00:00 2001 From: Huajun Qin Date: Thu, 26 Mar 2020 14:19:04 -0700 Subject: [PATCH 010/114] fix check-style build errors --- .../io/confluent/druid/transform/ExtractTransformTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java b/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java index 8c1d60c5fc99..6de448a233aa 100644 --- a/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java +++ b/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java @@ -4,7 +4,6 @@ package io.confluent.druid.transform; -import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -139,7 +138,7 @@ public void testNullTopic() public void testSerde() throws Exception { final TransformSpec transformSpec = new TransformSpec( - null, + null, ImmutableList.of( new ExtractTenantTopicTransform("tenant_topic", "topic"), new ExtractTenantTransform("tenant", "topic") From e69b515a816e2f779f4a86e76f0ee8c2f901074f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Wed, 25 Mar 2020 19:41:36 -0700 Subject: [PATCH 011/114] setup semaphore build --- .semaphore/semaphore.yml | 83 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 .semaphore/semaphore.yml diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml new file mode 100644 index 000000000000..cfeb4b046d3e --- /dev/null +++ b/.semaphore/semaphore.yml @@ -0,0 +1,83 @@ +version: v1.0 +name: Apache Druid +agent: + machine: + type: e1-standard-4 + os_image: ubuntu1804 + +blocks: + - name: "Install" + task: + env_vars: &env_vars + - name: MVN + value: "mvn -B" + - name: MAVEN_OPTS + value: "-Dmaven.repo.local=.m2" + - name: MAVEN_SKIP + value: > + -Danimal.sniffer.skip=true + -Dcheckstyle.skip=true + -Ddruid.console.skip=true + -Denforcer.skip=true + -Dforbiddenapis.skip=true + -Dmaven.javadoc.skip=true + -Dpmd.skip=true + -Dspotbugs.skip=true + + - name: MAVEN_SKIP_TESTS + value: "-DskipTests -Djacoco.skip=true" + prologue: + commands: + - sem-version java 8 + - checkout + jobs: + - name: "Install" + commands: + - cache restore + - > + MAVEN_OPTS="${MAVEN_OPTS} -Xmx3000m" ${MVN} clean install + -q -ff ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C + - cache store + + - name: "Tests" + task: + env_vars: *env_vars + prologue: + commands: + - sem-version java 8 + - checkout + - cache restore + jobs: + - name: "Confluent Extensions" + env_vars: + - name: MAVEN_PROJECTS + value: extensions-contrib/confluent-extensions + commands: &run_tests + - > + MAVEN_OPTS="${MAVEN_OPTS} -Xmx800m" ${MVN} test -pl ${MAVEN_PROJECTS} + ${MAVEN_SKIP} -Dremoteresources.skip=true + - name: "Server" + env_vars: + - name: MAVEN_PROJECTS + value: server + commands: *run_tests + - name: "Processing" + env_vars: + - name: MAVEN_PROJECTS + value: processing + commands: *run_tests + - name: "Indexing Service" + env_vars: + - name: MAVEN_PROJECTS + value: indexing-service + commands: *run_tests + - name: "Kafka Indexing Service" + env_vars: + - name: MAVEN_PROJECTS + value: extensions-core/kafka-indexing-service + commands: *run_tests + - name: "Other Tests" + env_vars: + - name: MAVEN_PROJECTS + value: '!server,!processing,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-contrib/confluent-extensions' + commands: *run_tests From f514bce2ba6fd6e9158d94f28f4132303aed067a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Thu, 26 Mar 2020 14:34:08 -0700 Subject: [PATCH 012/114] add checkstyle --- .semaphore/semaphore.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index cfeb4b046d3e..ef4e9737a9a6 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -48,6 +48,30 @@ blocks: - checkout - cache restore jobs: + - name: "animal sniffer checks" + commands: + - ${MVN} animal-sniffer:check --fail-at-end + + - name: "checkstyle" + commands: + - ${MVN} checkstyle:checkstyle --fail-at-end + + - name: "enforcer checks" + commands: + - ${MVN} enforcer:enforce --fail-at-end + + - name: "forbidden api checks" + commands: + - ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end + + - name: "pmd checks" + commands: + - ${MVN} pmd:check --fail-at-end # TODO: consider adding pmd:cpd-check + + - name: "spotbugs checks" + commands: + - ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks' + - name: "Confluent Extensions" env_vars: - name: MAVEN_PROJECTS @@ -56,26 +80,31 @@ blocks: - > MAVEN_OPTS="${MAVEN_OPTS} -Xmx800m" ${MVN} test -pl ${MAVEN_PROJECTS} ${MAVEN_SKIP} -Dremoteresources.skip=true + - name: "Server" env_vars: - name: MAVEN_PROJECTS value: server commands: *run_tests + - name: "Processing" env_vars: - name: MAVEN_PROJECTS value: processing commands: *run_tests + - name: "Indexing Service" env_vars: - name: MAVEN_PROJECTS value: indexing-service commands: *run_tests + - name: "Kafka Indexing Service" env_vars: - name: MAVEN_PROJECTS value: extensions-core/kafka-indexing-service commands: *run_tests + - name: "Other Tests" env_vars: - name: MAVEN_PROJECTS From 78e523ed200c7d7e7f094f7f15d8edbffe30ecfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Fri, 27 Mar 2020 09:25:15 -0700 Subject: [PATCH 013/114] fix edge cases for internal topics --- .../java/io/confluent/druid/transform/TenantUtils.java | 4 ++-- .../confluent/druid/transform/ExtractTransformTest.java | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/TenantUtils.java b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/TenantUtils.java index 3de3bcba641d..1a4e8c66df24 100644 --- a/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/TenantUtils.java +++ b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/TenantUtils.java @@ -14,13 +14,13 @@ public class TenantUtils public static String extractTenant(String prefixedTopic) { int i = prefixedTopic.indexOf(DELIMITER); - return i < 0 ? null : prefixedTopic.substring(0, i); + return i > 0 ? prefixedTopic.substring(0, i) : null; } @Nullable public static String extractTenantTopic(String prefixedTopic) { int i = prefixedTopic.indexOf(DELIMITER); - return i < 0 ? null : prefixedTopic.substring(i + 1); + return i > 0 ? prefixedTopic.substring(i + 1) : null; } } diff --git a/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java b/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java index 6de448a233aa..323b48b9c6ad 100644 --- a/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java +++ b/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java @@ -74,6 +74,15 @@ public void testExtraction() Assert.assertEquals(ImmutableList.of("mytopic"), row.getDimension("tenant_topic")); } + @Test + public void testInternal() + { + Assert.assertEquals(null, TenantUtils.extractTenantTopic("__consumer_offsets")); + Assert.assertEquals(null, TenantUtils.extractTenant("__consumer_offsets")); + Assert.assertEquals(null, TenantUtils.extractTenantTopic("other.topic")); + Assert.assertEquals(null, TenantUtils.extractTenant("other.topic")); + } + @Test public void testPreserveExistingFields() { From 419828a2cc5e27275f91af054eff63c9e5a86898 Mon Sep 17 00:00:00 2001 From: Apoorv Mittal Date: Fri, 24 Apr 2020 12:12:54 +0530 Subject: [PATCH 014/114] METRICS-1302: Added prefix support for resource labels. (#14) * METRICS-1302: Added prefix support for resource labels. * Addressed review comments. * Added and moved configs to ingestion spec, optimized code. * Addressed review comments * Updated metric dimesnion and other review comments * Flipped ternary operator * Moved from NullHandling to StringUtils. * Removed unnecessary HashMap. * Removed verbosity for instance variables. --- .../OpenCensusProtobufInputRowParser.java | 69 +++++++++----- .../OpenCensusProtobufInputRowParserTest.java | 89 ++++++++++++++++++- 2 files changed, 133 insertions(+), 25 deletions(-) diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java index 252cac590525..47d7f26c11bf 100644 --- a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.protobuf.ByteString; @@ -32,12 +33,14 @@ import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.ParseSpec; +import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.common.parsers.ParseException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -48,19 +51,31 @@ public class OpenCensusProtobufInputRowParser implements ByteBufferInputRowParse private static final Logger LOG = new Logger(OpenCensusProtobufInputRowParser.class); private static final String SEPARATOR = "-"; - public static final String NAME = "name"; - public static final String VALUE = "value"; - public static final String TIMESTAMP_COLUMN = "timestamp"; + private static final String DEFAULT_METRIC_DIMENSION = "name"; + private static final String VALUE = "value"; + private static final String TIMESTAMP_COLUMN = "timestamp"; + private static final String DEFAULT_RESOURCE_PREFIX = "resource."; private final ParseSpec parseSpec; private final List dimensions; + private final String metricDimension; + private final String metricLabelPrefix; + private final String resourceLabelPrefix; + @JsonCreator public OpenCensusProtobufInputRowParser( - @JsonProperty("parseSpec") ParseSpec parseSpec + @JsonProperty("parseSpec") ParseSpec parseSpec, + @JsonProperty("metricDimension") String metricDimension, + @JsonProperty("metricLabelPrefix") String metricPrefix, + @JsonProperty("resourceLabelPrefix") String resourcePrefix ) { this.parseSpec = parseSpec; this.dimensions = parseSpec.getDimensionsSpec().getDimensionNames(); + this.metricDimension = Strings.isNullOrEmpty(metricDimension) ? DEFAULT_METRIC_DIMENSION : metricDimension; + this.metricLabelPrefix = StringUtils.nullToEmptyNonDruidDataString(metricPrefix); + this.resourceLabelPrefix = resourcePrefix != null ? resourcePrefix : DEFAULT_RESOURCE_PREFIX; + LOG.info("Creating Open Census Protobuf parser with spec:" + parseSpec); } @@ -73,7 +88,11 @@ public ParseSpec getParseSpec() @Override public OpenCensusProtobufInputRowParser withParseSpec(ParseSpec parseSpec) { - return new OpenCensusProtobufInputRowParser(parseSpec); + return new OpenCensusProtobufInputRowParser( + parseSpec, + metricDimension, + metricLabelPrefix, + resourceLabelPrefix); } @Override @@ -88,22 +107,30 @@ public List parseBatch(ByteBuffer input) throw new ParseException(e, "Protobuf message could not be parsed"); } + // Process metric descriptor labels map keys. + List descriptorLabels = metric.getMetricDescriptor().getLabelKeysList().stream() + .map(s -> this.metricLabelPrefix + s.getKey()) + .collect(Collectors.toList()); + + // Process resource labels map. + Map resourceLabelsMap = metric.getResource().getLabelsMap().entrySet().stream() + .collect(Collectors.toMap(entry -> this.resourceLabelPrefix + entry.getKey(), + Map.Entry::getValue)); + final List dimensions; if (!this.dimensions.isEmpty()) { dimensions = this.dimensions; } else { - Set recordDimensions = metric.getMetricDescriptor().getLabelKeysList().stream() - .map(s -> s.getKey()) - .collect(Collectors.toSet()); + Set recordDimensions = new HashSet<>(descriptorLabels); // Add resource map key set to record dimensions. - recordDimensions.addAll(metric.getResource().getLabelsMap().keySet()); + recordDimensions.addAll(resourceLabelsMap.keySet()); - // NAME, VALUE dimensions will not be present in labelKeysList or Metric.Resource map as they - // are derived dimensions, which get populated while parsing data for timeSeries hence add - // them to recordDimensions. - recordDimensions.add(NAME); + // MetricDimension, VALUE dimensions will not be present in labelKeysList or Metric.Resource + // map as they are derived dimensions, which get populated while parsing data for timeSeries + // hence add them to recordDimensions. + recordDimensions.add(metricDimension); recordDimensions.add(VALUE); dimensions = Lists.newArrayList( @@ -116,11 +143,11 @@ public List parseBatch(ByteBuffer input) for (TimeSeries ts : metric.getTimeseriesList()) { // Add common resourceLabels. - Map labels = new HashMap<>(metric.getResource().getLabelsMap()); + Map labels = new HashMap<>(resourceLabelsMap); // Add labels to record. for (int i = 0; i < metric.getMetricDescriptor().getLabelKeysCount(); i++) { - labels.put(metric.getMetricDescriptor().getLabelKeys(i).getKey(), ts.getLabelValues(i).getValue()); + labels.put(descriptorLabels.get(i), ts.getLabelValues(i).getValue()); } // One row per timeSeries point. @@ -132,7 +159,7 @@ public List parseBatch(ByteBuffer input) case DOUBLE_VALUE: Map doubleGauge = new HashMap<>(); doubleGauge.putAll(labels); - doubleGauge.put(NAME, metric.getMetricDescriptor().getName()); + doubleGauge.put(metricDimension, metric.getMetricDescriptor().getName()); doubleGauge.put(VALUE, point.getDoubleValue()); addDerivedMetricsRow(doubleGauge, dimensions, rows); break; @@ -140,21 +167,21 @@ public List parseBatch(ByteBuffer input) HashMap intGauge = new HashMap<>(); intGauge.putAll(labels); intGauge.put(VALUE, point.getInt64Value()); - intGauge.put(NAME, metric.getMetricDescriptor().getName()); + intGauge.put(metricDimension, metric.getMetricDescriptor().getName()); addDerivedMetricsRow(intGauge, dimensions, rows); break; case SUMMARY_VALUE: // count Map summaryCount = new HashMap<>(); summaryCount.putAll(labels); - summaryCount.put(NAME, metric.getMetricDescriptor().getName() + SEPARATOR + "count"); + summaryCount.put(metricDimension, metric.getMetricDescriptor().getName() + SEPARATOR + "count"); summaryCount.put(VALUE, point.getSummaryValue().getCount().getValue()); addDerivedMetricsRow(summaryCount, dimensions, rows); // sum Map summarySum = new HashMap<>(); summarySum.putAll(labels); - summarySum.put(NAME, metric.getMetricDescriptor().getName() + SEPARATOR + "sum"); + summarySum.put(metricDimension, metric.getMetricDescriptor().getName() + SEPARATOR + "sum"); summarySum.put(VALUE, point.getSummaryValue().getSnapshot().getSum().getValue()); addDerivedMetricsRow(summarySum, dimensions, rows); @@ -163,13 +190,13 @@ public List parseBatch(ByteBuffer input) case DISTRIBUTION_VALUE: // count Map distCount = new HashMap<>(); - distCount.put(NAME, metric.getMetricDescriptor().getName() + SEPARATOR + "count"); + distCount.put(metricDimension, metric.getMetricDescriptor().getName() + SEPARATOR + "count"); distCount.put(VALUE, point.getDistributionValue().getCount()); addDerivedMetricsRow(distCount, dimensions, rows); // sum Map distSum = new HashMap<>(); - distSum.put(NAME, metric.getMetricDescriptor().getName() + SEPARATOR + "sum"); + distSum.put(metricDimension, metric.getMetricDescriptor().getName() + SEPARATOR + "sum"); distSum.put(VALUE, point.getDistributionValue().getSum()); addDerivedMetricsRow(distSum, dimensions, rows); // TODO: How to handle buckets ? diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java index 88a422585495..487d28988ee4 100644 --- a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java @@ -101,7 +101,7 @@ public void testGaugeParse() throws Exception { //configure parser with desc file - OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec); + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, null, null, ""); DateTime dateTime = new DateTime(2019, 07, 12, 9, 30, ISOChronology.getInstanceUTC()); @@ -128,7 +128,7 @@ public void testGaugeParse() throws Exception public void testSummaryParse() throws Exception { //configure parser with desc file - OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec); + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, null, null, ""); DateTime dateTime = new DateTime(2019, 07, 12, 9, 30, ISOChronology.getInstanceUTC()); @@ -161,7 +161,7 @@ public void testSummaryParse() throws Exception public void testDimensionsParseWithParseSpecDimensions() throws Exception { //configure parser with desc file - OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpecWithDimensions); + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpecWithDimensions, null, null, ""); DateTime dateTime = new DateTime(2019, 07, 12, 9, 30, ISOChronology.getInstanceUTC()); @@ -192,7 +192,7 @@ public void testDimensionsParseWithParseSpecDimensions() throws Exception public void testDimensionsParseWithoutParseSpecDimensions() throws Exception { //configure parser with desc file - OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec); + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, null, null, ""); DateTime dateTime = new DateTime(2019, 07, 12, 9, 30, ISOChronology.getInstanceUTC()); @@ -221,6 +221,87 @@ public void testDimensionsParseWithoutParseSpecDimensions() throws Exception } + @Test + public void testMetricNameOverride() throws Exception + { + //configure parser with desc file + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, "dimension_name", null, ""); + + Metric metric = summaryMetric(Timestamp.getDefaultInstance()); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + metric.writeTo(out); + + List rows = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())); + + Assert.assertEquals(2, rows.size()); + + InputRow row = rows.get(0); + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "dimension_name", "metric_summary-count"); + assertDimensionEquals(row, "foo_key", "foo_value"); + assertDimensionEquals(row, "env_key", "env_val"); + + row = rows.get(1); + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "dimension_name", "metric_summary-sum"); + assertDimensionEquals(row, "foo_key", "foo_value"); + assertDimensionEquals(row, "env_key", "env_val"); + } + + @Test + public void testDefaultPrefix() throws Exception + { + //configure parser with desc file + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, null, null, null); + + Metric metric = summaryMetric(Timestamp.getDefaultInstance()); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + metric.writeTo(out); + + List rows = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())); + + Assert.assertEquals(2, rows.size()); + + InputRow row = rows.get(0); + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "name", "metric_summary-count"); + assertDimensionEquals(row, "foo_key", "foo_value"); + assertDimensionEquals(row, "resource.env_key", "env_val"); + + row = rows.get(1); + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "name", "metric_summary-sum"); + assertDimensionEquals(row, "foo_key", "foo_value"); + assertDimensionEquals(row, "resource.env_key", "env_val"); + } + + @Test + public void testCustomPrefix() throws Exception + { + //configure parser with desc file + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, null, "descriptor.", "custom."); + + Metric metric = summaryMetric(Timestamp.getDefaultInstance()); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + metric.writeTo(out); + + List rows = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())); + + Assert.assertEquals(2, rows.size()); + + InputRow row = rows.get(0); + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "name", "metric_summary-count"); + assertDimensionEquals(row, "descriptor.foo_key", "foo_value"); + assertDimensionEquals(row, "custom.env_key", "env_val"); + + row = rows.get(1); + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "name", "metric_summary-sum"); + assertDimensionEquals(row, "descriptor.foo_key", "foo_value"); + assertDimensionEquals(row, "custom.env_key", "env_val"); + } + private void assertDimensionEquals(InputRow row, String dimension, Object expected) { List values = row.getDimension(dimension); From 2ca99d78ef185ca95586c0851b4a14c1f38cb599 Mon Sep 17 00:00:00 2001 From: Apoorv Mittal Date: Tue, 5 May 2020 13:02:53 +0530 Subject: [PATCH 015/114] Added getters for configs, labels for distribution metric. (#15) * Added getters for configs, labels for distribution metric. * Addressed review comments * Removed extra brackets in JsonProperty. --- .../OpenCensusProtobufInputRowParser.java | 45 ++++- .../OpenCensusProtobufInputRowParserTest.java | 178 +++++++++++++----- 2 files changed, 173 insertions(+), 50 deletions(-) diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java index 47d7f26c11bf..1676e82abb03 100644 --- a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java @@ -43,6 +43,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -85,6 +86,24 @@ public ParseSpec getParseSpec() return parseSpec; } + @JsonProperty + public String getMetricDimension() + { + return metricDimension; + } + + @JsonProperty + public String getMetricLabelPrefix() + { + return metricLabelPrefix; + } + + @JsonProperty + public String getResourceLabelPrefix() + { + return resourceLabelPrefix; + } + @Override public OpenCensusProtobufInputRowParser withParseSpec(ParseSpec parseSpec) { @@ -113,7 +132,7 @@ public List parseBatch(ByteBuffer input) .collect(Collectors.toList()); // Process resource labels map. - Map resourceLabelsMap = metric.getResource().getLabelsMap().entrySet().stream() + Map resourceLabelsMap = metric.getResource().getLabelsMap().entrySet().stream() .collect(Collectors.toMap(entry -> this.resourceLabelPrefix + entry.getKey(), Map.Entry::getValue)); @@ -190,12 +209,14 @@ public List parseBatch(ByteBuffer input) case DISTRIBUTION_VALUE: // count Map distCount = new HashMap<>(); + distCount.putAll(labels); distCount.put(metricDimension, metric.getMetricDescriptor().getName() + SEPARATOR + "count"); distCount.put(VALUE, point.getDistributionValue().getCount()); addDerivedMetricsRow(distCount, dimensions, rows); // sum Map distSum = new HashMap<>(); + distSum.putAll(labels); distSum.put(metricDimension, metric.getMetricDescriptor().getName() + SEPARATOR + "sum"); distSum.put(VALUE, point.getDistributionValue().getSum()); addDerivedMetricsRow(distSum, dimensions, rows); @@ -218,4 +239,26 @@ private void addDerivedMetricsRow(Map derivedMetrics, List rows = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())); + + Assert.assertEquals(2, rows.size()); + + InputRow row = rows.get(0); + Assert.assertEquals(dateTime.getMillis(), row.getTimestampFromEpoch()); + assertDimensionEquals(row, "name", "metric_distribution-count"); + assertDimensionEquals(row, "foo_key", "foo_value"); + Assert.assertEquals(100, row.getMetric("value").intValue()); + + row = rows.get(1); + Assert.assertEquals(dateTime.getMillis(), row.getTimestampFromEpoch()); + assertDimensionEquals(row, "name", "metric_distribution-sum"); + assertDimensionEquals(row, "foo_key", "foo_value"); + Assert.assertEquals(500, row.getMetric("value").doubleValue(), 0.0); } @Test @@ -302,6 +358,20 @@ public void testCustomPrefix() throws Exception assertDimensionEquals(row, "custom.env_key", "env_val"); } + @Test + public void testSerde() throws Exception + { + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, "metric.name", "descriptor.", "custom."); + + final ObjectMapper jsonMapper = new ObjectMapper(); + jsonMapper.registerModules(new OpenCensusProtobufExtensionsModule().getJacksonModules()); + + Assert.assertEquals(parser, jsonMapper.readValue( + jsonMapper.writeValueAsString(parser), + ByteBufferInputRowParser.class + )); + } + private void assertDimensionEquals(InputRow row, String dimension, Object expected) { List values = row.getDimension(dimension); @@ -312,43 +382,29 @@ private void assertDimensionEquals(InputRow row, String dimension, Object expect private Metric doubleGaugeMetric(Timestamp timestamp) { - Metric dist = Metric.newBuilder() - .setMetricDescriptor( - MetricDescriptor.newBuilder() - .setName("metric_gauge_double") - .setDescription("metric_gauge_double_description") - .setUnit("ms") - .setType( - MetricDescriptor.Type.GAUGE_DOUBLE) - .addLabelKeys( - LabelKey.newBuilder() - .setKey("foo_key") - .build()) - .build()) - .setResource( - Resource.newBuilder() - .setType("env") - .putAllLabels(Collections.singletonMap("env_key", "env_val")) - .build()) - .addTimeseries( - TimeSeries.newBuilder() - .setStartTimestamp(timestamp) - .addLabelValues( - LabelValue.newBuilder() - .setHasValue(true) - .setValue("foo_value") - .build()) - .addPoints( - Point.newBuilder() - .setTimestamp(timestamp) - .setDoubleValue(2000) - .build()) - .build()) - .build(); - - return dist; + return getMetric( + "metric_gauge_double", + "metric_gauge_double_description", + Type.GAUGE_DOUBLE, + Point.newBuilder() + .setTimestamp(timestamp) + .setDoubleValue(2000) + .build(), + timestamp); } + private Metric intGaugeMetric(Timestamp timestamp) + { + return getMetric( + "metric_gauge_int64", + "metric_gauge_int64_description", + MetricDescriptor.Type.GAUGE_INT64, + Point.newBuilder() + .setTimestamp(timestamp) + .setInt64Value(1000) + .build(), + timestamp); + } private Metric summaryMetric(Timestamp timestamp) { @@ -387,15 +443,44 @@ private Metric summaryMetric(Timestamp timestamp) .setSnapshot(snapshot) .build(); + return getMetric( + "metric_summary", + "metric_summary_description", + MetricDescriptor.Type.SUMMARY, + Point.newBuilder() + .setTimestamp(timestamp) + .setSummaryValue(summaryValue) + .build(), + timestamp); + } + + private Metric distributionMetric(Timestamp timestamp) + { + DistributionValue distributionValue = DistributionValue.newBuilder() + .setCount(100) + .setSum(500) + .build(); + return getMetric( + "metric_distribution", + "metric_distribution_description", + MetricDescriptor.Type.GAUGE_DISTRIBUTION, + Point.newBuilder() + .setTimestamp(timestamp) + .setDistributionValue(distributionValue) + .build(), + timestamp); + } + + private Metric getMetric(String name, String description, MetricDescriptor.Type type, Point point, Timestamp timestamp) + { Metric dist = Metric.newBuilder() .setMetricDescriptor( MetricDescriptor.newBuilder() - .setName("metric_summary") - .setDescription("metric_summary_description") + .setName(name) + .setDescription(description) .setUnit("ms") - .setType( - MetricDescriptor.Type.SUMMARY) + .setType(type) .addLabelKeys( LabelKey.newBuilder() .setKey("foo_key") @@ -414,16 +499,11 @@ private Metric summaryMetric(Timestamp timestamp) .setHasValue(true) .setValue("foo_value") .build()) - .addPoints( - Point.newBuilder() - .setTimestamp(timestamp) - .setSummaryValue(summaryValue) - .build()) + .addPoints(point) .build()) .build(); return dist; } - } From d810b3bfdc8d25f11c7a5fc1f1db65f021c46c35 Mon Sep 17 00:00:00 2001 From: Apoorv Mittal Date: Thu, 7 May 2020 21:56:38 +0530 Subject: [PATCH 016/114] Default resource label prefix to blank - Backward Compatibility (#16) --- .../opencensus/protobuf/OpenCensusProtobufInputRowParser.java | 2 +- .../protobuf/OpenCensusProtobufInputRowParserTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java index 1676e82abb03..d7e337997c51 100644 --- a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java @@ -55,7 +55,7 @@ public class OpenCensusProtobufInputRowParser implements ByteBufferInputRowParse private static final String DEFAULT_METRIC_DIMENSION = "name"; private static final String VALUE = "value"; private static final String TIMESTAMP_COLUMN = "timestamp"; - private static final String DEFAULT_RESOURCE_PREFIX = "resource."; + private static final String DEFAULT_RESOURCE_PREFIX = ""; private final ParseSpec parseSpec; private final List dimensions; diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java index eab42b744b11..2501f115bbdd 100644 --- a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java @@ -322,13 +322,13 @@ public void testDefaultPrefix() throws Exception Assert.assertEquals(4, row.getDimensions().size()); assertDimensionEquals(row, "name", "metric_summary-count"); assertDimensionEquals(row, "foo_key", "foo_value"); - assertDimensionEquals(row, "resource.env_key", "env_val"); + assertDimensionEquals(row, "env_key", "env_val"); row = rows.get(1); Assert.assertEquals(4, row.getDimensions().size()); assertDimensionEquals(row, "name", "metric_summary-sum"); assertDimensionEquals(row, "foo_key", "foo_value"); - assertDimensionEquals(row, "resource.env_key", "env_val"); + assertDimensionEquals(row, "env_key", "env_val"); } @Test From cf5fe23ed2eec90135145425676c2f9a8175fc9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 18 Aug 2020 15:11:09 -0700 Subject: [PATCH 017/114] update opencensus parent pom version --- extensions-contrib/opencensus-extensions/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml index 503f81925f80..9c1611230478 100644 --- a/extensions-contrib/opencensus-extensions/pom.xml +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -31,7 +31,7 @@ druid org.apache.druid - 0.17.0 + 0.19.0 ../../pom.xml From aee6405eafb4f2da63c0d72860e01d4ac65cd340 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 18 Aug 2020 15:35:51 -0700 Subject: [PATCH 018/114] update opencensus extensions for 0.19.x --- .../protobuf/OpenCensusProtobufInputRowParserTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java index 2501f115bbdd..9e0cf6881f53 100644 --- a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java @@ -80,7 +80,7 @@ public void setUp() new JSONPathFieldSpec(JSONPathFieldType.ROOT, "value", ""), new JSONPathFieldSpec(JSONPathFieldType.ROOT, "foo_key", "") ) - ), null + ), null, null ); parseSpecWithDimensions = new JSONParseSpec( @@ -95,7 +95,7 @@ public void setUp() new JSONPathFieldSpec(JSONPathFieldType.ROOT, "value", ""), new JSONPathFieldSpec(JSONPathFieldType.ROOT, "foo_key", "") ) - ), null + ), null, null ); } From 473919f495ae174ae1e556ed773acf0e26e8ff34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 18 Aug 2020 15:43:52 -0700 Subject: [PATCH 019/114] update parent pom version for confluent-extensions --- extensions-contrib/confluent-extensions/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-contrib/confluent-extensions/pom.xml b/extensions-contrib/confluent-extensions/pom.xml index 0c640d272185..ad11d670afa7 100644 --- a/extensions-contrib/confluent-extensions/pom.xml +++ b/extensions-contrib/confluent-extensions/pom.xml @@ -17,7 +17,7 @@ druid org.apache.druid - 0.17.0 + 0.19.0 ../../pom.xml From a3701f22436f6c454f9c9ba07313d8e84fbd04d7 Mon Sep 17 00:00:00 2001 From: Huajun Qin Date: Fri, 11 Sep 2020 10:24:19 -0700 Subject: [PATCH 020/114] Add the capability to speed up S3 uploads using AWS transfer manager --- .../druid/storage/s3/S3StorageConfig.java | 18 ++++- .../druid/storage/s3/S3TransferConfig.java | 71 +++++++++++++++++++ .../org/apache/druid/storage/s3/S3Utils.java | 7 +- .../s3/ServerSideEncryptingAmazonS3.java | 29 +++++++- .../data/input/s3/S3InputSourceTest.java | 4 +- .../storage/s3/ObjectSummaryIteratorTest.java | 2 +- .../storage/s3/S3DataSegmentArchiverTest.java | 3 +- .../storage/s3/S3DataSegmentMoverTest.java | 2 +- .../storage/s3/S3DataSegmentPusherTest.java | 7 +- .../s3/S3StorageConnectorProviderTest.java | 2 +- .../druid/storage/s3/S3TaskLogsTest.java | 11 ++- .../s3/TestAWSCredentialsProvider.java | 4 +- .../output/RetryableS3OutputStreamTest.java | 3 +- .../s3/output/S3StorageConnectorTest.java | 4 +- 14 files changed, 141 insertions(+), 26 deletions(-) create mode 100644 extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3TransferConfig.java diff --git a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageConfig.java b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageConfig.java index cfae0eb084b7..b52d13cd518e 100644 --- a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageConfig.java +++ b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageConfig.java @@ -36,12 +36,22 @@ public class S3StorageConfig @JsonProperty("sse") private final ServerSideEncryption serverSideEncryption; + /** + * S3 transfer config. + * + * @see S3StorageDruidModule#configure + */ + @JsonProperty("transfer") + private final S3TransferConfig s3TransferConfig; + @JsonCreator public S3StorageConfig( - @JsonProperty("sse") ServerSideEncryption serverSideEncryption + @JsonProperty("sse") ServerSideEncryption serverSideEncryption, + @JsonProperty("transfer") S3TransferConfig s3TransferConfig ) { this.serverSideEncryption = serverSideEncryption == null ? new NoopServerSideEncryption() : serverSideEncryption; + this.s3TransferConfig = s3TransferConfig == null ? new S3TransferConfig() : s3TransferConfig; } @JsonProperty("sse") @@ -49,4 +59,10 @@ public ServerSideEncryption getServerSideEncryption() { return serverSideEncryption; } + + @JsonProperty("transfer") + public S3TransferConfig getS3TransferConfig() + { + return s3TransferConfig; + } } diff --git a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3TransferConfig.java b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3TransferConfig.java new file mode 100644 index 000000000000..fc8bd8903fad --- /dev/null +++ b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3TransferConfig.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.storage.s3; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.Min; + +/** + */ +public class S3TransferConfig +{ + @JsonProperty + private boolean useTransferManager = false; + + @JsonProperty + @Min(1) + private long minimumUploadPartSize = 5 * 1024 * 1024L; + + @JsonProperty + @Min(1) + private long multipartUploadThreshold = 5 * 1024 * 1024L; + + public void setUseTransferManager(boolean useTransferManager) + { + this.useTransferManager = useTransferManager; + } + + public void setMinimumUploadPartSize(long minimumUploadPartSize) + { + this.minimumUploadPartSize = minimumUploadPartSize; + } + + public void setMultipartUploadThreshold(long multipartUploadThreshold) + { + this.multipartUploadThreshold = multipartUploadThreshold; + } + + public boolean isUseTransferManager() + { + return useTransferManager; + } + + public long getMinimumUploadPartSize() + { + return minimumUploadPartSize; + } + + public long getMultipartUploadThreshold() + { + return multipartUploadThreshold; + } + +} diff --git a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java index 087d6684c7f3..a256b7001506 100644 --- a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java +++ b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java @@ -83,6 +83,9 @@ public boolean apply(Throwable e) } else if (e instanceof SdkClientException && e.getMessage().contains("Unable to execute HTTP request")) { // This is likely due to a temporary DNS issue and can be retried. return true; + } else if (e instanceof InterruptedException) { + Thread.interrupted(); // Clear interrupted state and not retry + return false; } else if (e instanceof AmazonClientException) { return AWSClientUtil.isClientExceptionRecoverable((AmazonClientException) e); } else { @@ -320,7 +323,7 @@ static void uploadFileIfPossible( String bucket, String key, File file - ) + ) throws InterruptedException { final PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, file); @@ -328,7 +331,7 @@ static void uploadFileIfPossible( putObjectRequest.setAccessControlList(S3Utils.grantFullControlToBucketOwner(service, bucket)); } log.info("Pushing [%s] to bucket[%s] and key[%s].", file, bucket, key); - service.putObject(putObjectRequest); + service.upload(putObjectRequest); } @Nullable diff --git a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/ServerSideEncryptingAmazonS3.java b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/ServerSideEncryptingAmazonS3.java index 320a0b9a6f99..d97d8df6c8a8 100644 --- a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/ServerSideEncryptingAmazonS3.java +++ b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/ServerSideEncryptingAmazonS3.java @@ -43,6 +43,9 @@ import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.UploadPartRequest; import com.amazonaws.services.s3.model.UploadPartResult; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.TransferManagerBuilder; +import com.amazonaws.services.s3.transfer.Upload; import org.apache.druid.java.util.common.ISE; import java.io.File; @@ -65,11 +68,21 @@ public static Builder builder() private final AmazonS3 amazonS3; private final ServerSideEncryption serverSideEncryption; + private final TransferManager transferManager; - public ServerSideEncryptingAmazonS3(AmazonS3 amazonS3, ServerSideEncryption serverSideEncryption) + public ServerSideEncryptingAmazonS3(AmazonS3 amazonS3, ServerSideEncryption serverSideEncryption, S3TransferConfig transferConfig) { this.amazonS3 = amazonS3; this.serverSideEncryption = serverSideEncryption; + if (transferConfig.isUseTransferManager()) { + this.transferManager = TransferManagerBuilder.standard() + .withS3Client(amazonS3) + .withMinimumUploadPartSize(transferConfig.getMinimumUploadPartSize()) + .withMultipartUploadThreshold(transferConfig.getMultipartUploadThreshold()) + .build(); + } else { + this.transferManager = null; + } } public boolean doesObjectExist(String bucket, String objectName) @@ -168,10 +181,20 @@ public CompleteMultipartUploadResult completeMultipartUpload(CompleteMultipartUp return amazonS3.completeMultipartUpload(request); } + public void upload(PutObjectRequest request) throws InterruptedException + { + if (transferManager == null) { + putObject(request); + } else { + Upload transfer = transferManager.upload(serverSideEncryption.decorate(request)); + transfer.waitForCompletion(); + } + } + public static class Builder { private AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3Client.builder(); - private S3StorageConfig s3StorageConfig = new S3StorageConfig(new NoopServerSideEncryption()); + private S3StorageConfig s3StorageConfig = new S3StorageConfig(new NoopServerSideEncryption(), null); public Builder setAmazonS3ClientBuilder(AmazonS3ClientBuilder amazonS3ClientBuilder) { @@ -204,7 +227,7 @@ public ServerSideEncryptingAmazonS3 build() throw new ISE("S3StorageConfig cannot be null!"); } - return new ServerSideEncryptingAmazonS3(amazonS3ClientBuilder.build(), s3StorageConfig.getServerSideEncryption()); + return new ServerSideEncryptingAmazonS3(amazonS3ClientBuilder.build(), s3StorageConfig.getServerSideEncryption(), s3StorageConfig.getS3TransferConfig()); } } } diff --git a/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java b/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java index 6b0bb537c7e1..9daf4cb42860 100644 --- a/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java +++ b/extensions-core/s3-extensions/src/test/java/org/apache/druid/data/input/s3/S3InputSourceTest.java @@ -70,6 +70,7 @@ import org.apache.druid.metadata.DefaultPasswordProvider; import org.apache.druid.storage.s3.NoopServerSideEncryption; import org.apache.druid.storage.s3.S3InputDataConfig; +import org.apache.druid.storage.s3.S3TransferConfig; import org.apache.druid.storage.s3.S3Utils; import org.apache.druid.storage.s3.ServerSideEncryptingAmazonS3; import org.apache.druid.testing.InitializedNullHandlingTest; @@ -108,7 +109,8 @@ public class S3InputSourceTest extends InitializedNullHandlingTest public static final AmazonS3ClientBuilder AMAZON_S3_CLIENT_BUILDER = AmazonS3Client.builder(); public static final ServerSideEncryptingAmazonS3 SERVICE = new ServerSideEncryptingAmazonS3( S3_CLIENT, - new NoopServerSideEncryption() + new NoopServerSideEncryption(), + new S3TransferConfig() ); public static final S3InputDataConfig INPUT_DATA_CONFIG; private static final int MAX_LISTING_LENGTH = 10; diff --git a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/ObjectSummaryIteratorTest.java b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/ObjectSummaryIteratorTest.java index ea2ca4af26c1..8ee6c826718d 100644 --- a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/ObjectSummaryIteratorTest.java +++ b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/ObjectSummaryIteratorTest.java @@ -195,7 +195,7 @@ private static ServerSideEncryptingAmazonS3 makeMockClient( final List objects ) { - return new ServerSideEncryptingAmazonS3(null, null) + return new ServerSideEncryptingAmazonS3(null, null, new S3TransferConfig()) { @Override public ListObjectsV2Result listObjectsV2(final ListObjectsV2Request request) diff --git a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentArchiverTest.java b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentArchiverTest.java index f5005c706e01..4acf553fae50 100644 --- a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentArchiverTest.java +++ b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentArchiverTest.java @@ -76,7 +76,8 @@ public String getArchiveBaseKey() private static final Supplier S3_SERVICE = Suppliers.ofInstance( new ServerSideEncryptingAmazonS3( EasyMock.createStrictMock(AmazonS3Client.class), - new NoopServerSideEncryption() + new NoopServerSideEncryption(), + new S3TransferConfig() ) ); private static final S3DataSegmentPuller PULLER = new S3DataSegmentPuller(S3_SERVICE.get()); diff --git a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentMoverTest.java b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentMoverTest.java index 550a72cef43c..8f653e956a83 100644 --- a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentMoverTest.java +++ b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentMoverTest.java @@ -201,7 +201,7 @@ private static class MockAmazonS3Client extends ServerSideEncryptingAmazonS3 private MockAmazonS3Client() { - super(new AmazonS3Client(), new NoopServerSideEncryption()); + super(new AmazonS3Client(), new NoopServerSideEncryption(), new S3TransferConfig()); } public boolean didMove() diff --git a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentPusherTest.java b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentPusherTest.java index 23ab725aa33d..449fad012ccc 100644 --- a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentPusherTest.java +++ b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3DataSegmentPusherTest.java @@ -24,7 +24,7 @@ import com.amazonaws.services.s3.model.Grant; import com.amazonaws.services.s3.model.Owner; import com.amazonaws.services.s3.model.Permission; -import com.amazonaws.services.s3.model.PutObjectResult; +import com.amazonaws.services.s3.model.PutObjectRequest; import com.google.common.io.Files; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.timeline.DataSegment; @@ -83,9 +83,8 @@ private void testPushInternal(boolean useUniquePath, String matcher) throws Exce acl.grantAllPermissions(new Grant(new CanonicalGrantee(acl.getOwner().getId()), Permission.FullControl)); EasyMock.expect(s3Client.getBucketAcl(EasyMock.eq("bucket"))).andReturn(acl).once(); - EasyMock.expect(s3Client.putObject(EasyMock.anyObject())) - .andReturn(new PutObjectResult()) - .once(); + s3Client.upload(EasyMock.anyObject(PutObjectRequest.class)); + EasyMock.expectLastCall().once(); EasyMock.replay(s3Client); diff --git a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3StorageConnectorProviderTest.java b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3StorageConnectorProviderTest.java index 9f9d632f6181..790a4f1a2643 100644 --- a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3StorageConnectorProviderTest.java +++ b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3StorageConnectorProviderTest.java @@ -145,7 +145,7 @@ public void configure(Binder binder) new InjectableValues.Std() .addValue( ServerSideEncryptingAmazonS3.class, - new ServerSideEncryptingAmazonS3(null, new NoopServerSideEncryption()) + new ServerSideEncryptingAmazonS3(null, new NoopServerSideEncryption(), new S3TransferConfig()) )); diff --git a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3TaskLogsTest.java b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3TaskLogsTest.java index 011dc4888456..bd0067ee9dfe 100644 --- a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3TaskLogsTest.java +++ b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3TaskLogsTest.java @@ -29,7 +29,6 @@ import com.amazonaws.services.s3.model.Owner; import com.amazonaws.services.s3.model.Permission; import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.PutObjectResult; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.google.common.base.Optional; @@ -617,9 +616,8 @@ private S3TaskLogs getS3TaskLogs() private List testPushInternal(boolean disableAcl, String ownerId, String ownerDisplayName) throws Exception { - EasyMock.expect(s3Client.putObject(EasyMock.anyObject())) - .andReturn(new PutObjectResult()) - .once(); + s3Client.upload(EasyMock.anyObject(PutObjectRequest.class)); + EasyMock.expectLastCall().once(); AccessControlList aclExpected = new AccessControlList(); aclExpected.setOwner(new Owner(ownerId, ownerDisplayName)); @@ -628,9 +626,8 @@ private List testPushInternal(boolean disableAcl, String ownerId, String .andReturn(aclExpected) .once(); - EasyMock.expect(s3Client.putObject(EasyMock.anyObject(PutObjectRequest.class))) - .andReturn(new PutObjectResult()) - .once(); + s3Client.upload(EasyMock.anyObject(PutObjectRequest.class)); + EasyMock.expectLastCall().once(); EasyMock.replay(s3Client); diff --git a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/TestAWSCredentialsProvider.java b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/TestAWSCredentialsProvider.java index 3685fc6fa19b..fefcb8c3c38b 100644 --- a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/TestAWSCredentialsProvider.java +++ b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/TestAWSCredentialsProvider.java @@ -67,7 +67,7 @@ public void testWithFixedAWSKeys() new AWSProxyConfig(), new AWSEndpointConfig(), new AWSClientConfig(), - new S3StorageConfig(new NoopServerSideEncryption()) + new S3StorageConfig(new NoopServerSideEncryption(), null) ); s3Module.getAmazonS3Client( @@ -102,7 +102,7 @@ public void testWithFileSessionCredentials() throws IOException new AWSProxyConfig(), new AWSEndpointConfig(), new AWSClientConfig(), - new S3StorageConfig(new NoopServerSideEncryption()) + new S3StorageConfig(new NoopServerSideEncryption(), null) ); s3Module.getAmazonS3Client( diff --git a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/output/RetryableS3OutputStreamTest.java b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/output/RetryableS3OutputStreamTest.java index 1f8eac3bbae0..f407c2a41d7e 100644 --- a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/output/RetryableS3OutputStreamTest.java +++ b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/output/RetryableS3OutputStreamTest.java @@ -34,6 +34,7 @@ import org.apache.druid.java.util.common.IOE; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.storage.s3.NoopServerSideEncryption; +import org.apache.druid.storage.s3.S3TransferConfig; import org.apache.druid.storage.s3.ServerSideEncryptingAmazonS3; import org.easymock.EasyMock; import org.hamcrest.CoreMatchers; @@ -228,7 +229,7 @@ private static class TestAmazonS3 extends ServerSideEncryptingAmazonS3 private TestAmazonS3(int totalUploadFailures) { - super(EasyMock.createMock(AmazonS3.class), new NoopServerSideEncryption()); + super(EasyMock.createMock(AmazonS3.class), new NoopServerSideEncryption(), new S3TransferConfig()); this.uploadFailuresLeft = totalUploadFailures; } diff --git a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/output/S3StorageConnectorTest.java b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/output/S3StorageConnectorTest.java index 380c5cb1e508..a2f4d7e1a459 100644 --- a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/output/S3StorageConnectorTest.java +++ b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/output/S3StorageConnectorTest.java @@ -33,6 +33,7 @@ import com.google.common.collect.Lists; import org.apache.druid.storage.StorageConnector; import org.apache.druid.storage.s3.NoopServerSideEncryption; +import org.apache.druid.storage.s3.S3TransferConfig; import org.apache.druid.storage.s3.ServerSideEncryptingAmazonS3; import org.easymock.Capture; import org.easymock.EasyMock; @@ -64,7 +65,8 @@ public class S3StorageConnectorTest private final AmazonS3Client s3Client = EasyMock.createMock(AmazonS3Client.class); private final ServerSideEncryptingAmazonS3 service = new ServerSideEncryptingAmazonS3( s3Client, - new NoopServerSideEncryption() + new NoopServerSideEncryption(), + new S3TransferConfig() ); private final ListObjectsV2Result testResult = EasyMock.createMock(ListObjectsV2Result.class); From ace65fc814df93167d88fd42e1fd8e24df324932 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Mon, 21 Dec 2020 13:17:41 -0800 Subject: [PATCH 021/114] fix conflicting protobuf dependencies Align protobuf dependencies to use the main pom one --- extensions-contrib/opencensus-extensions/pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml index 9c1611230478..84d0aeb91ebc 100644 --- a/extensions-contrib/opencensus-extensions/pom.xml +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -35,10 +35,6 @@ ../../pom.xml - - 3.2.0 - - io.opencensus From b8e43ed9e4d9023613a38e839fd9d98da2d8956f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Wed, 16 Dec 2020 13:55:58 -0800 Subject: [PATCH 022/114] fix timestamp milliseconds in OpenCensusProtobufInputRowParser - fix millisecond resolution being dropped when converting timestamps - remove unnecessary conversion of ByteBuffer to ByteString - make test code a little more concise --- .../OpenCensusProtobufInputRowParser.java | 9 +- .../OpenCensusProtobufInputRowParserTest.java | 116 ++++++------------ 2 files changed, 41 insertions(+), 84 deletions(-) diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java index d7e337997c51..24a36e3a190f 100644 --- a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java @@ -24,8 +24,8 @@ import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Timestamp; import io.opencensus.proto.metrics.v1.Metric; import io.opencensus.proto.metrics.v1.Point; import io.opencensus.proto.metrics.v1.TimeSeries; @@ -38,6 +38,7 @@ import org.apache.druid.java.util.common.parsers.ParseException; import java.nio.ByteBuffer; +import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -120,7 +121,7 @@ public List parseBatch(ByteBuffer input) Metric metric; try { - metric = Metric.parseFrom(ByteString.copyFrom(input)); + metric = Metric.parseFrom(input); } catch (InvalidProtocolBufferException e) { throw new ParseException(e, "Protobuf message could not be parsed"); @@ -172,7 +173,9 @@ public List parseBatch(ByteBuffer input) // One row per timeSeries point. for (Point point : ts.getPointsList()) { // Time in millis - labels.put(TIMESTAMP_COLUMN, point.getTimestamp().getSeconds() * 1000); + final Timestamp t = point.getTimestamp(); + final long millis = Instant.ofEpochSecond(t.getSeconds(), t.getNanos()).toEpochMilli(); + labels.put(TIMESTAMP_COLUMN, millis); switch (point.getValueCase()) { case DOUBLE_VALUE: diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java index 9e0cf6881f53..503dc754a3cc 100644 --- a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java @@ -45,21 +45,23 @@ import org.apache.druid.java.util.common.parsers.JSONPathFieldSpec; import org.apache.druid.java.util.common.parsers.JSONPathFieldType; import org.apache.druid.java.util.common.parsers.JSONPathSpec; -import org.joda.time.DateTime; -import org.joda.time.chrono.ISOChronology; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; +import java.time.Instant; import java.util.Collections; import java.util.List; public class OpenCensusProtobufInputRowParserTest { + private static final Instant INSTANT = Instant.parse("2019-07-12T09:30:01.123Z"); + private static final Timestamp TIMESTAMP = Timestamp.newBuilder() + .setSeconds(INSTANT.getEpochSecond()) + .setNanos(INSTANT.getNano()).build(); @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -101,22 +103,15 @@ public void setUp() @Test - public void testDoubleGaugeParse() throws Exception + public void testDoubleGaugeParse() { //configure parser with desc file OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, null, null, ""); - DateTime dateTime = new DateTime(2019, 07, 12, 9, 30, ISOChronology.getInstanceUTC()); + Metric metric = doubleGaugeMetric(TIMESTAMP); - Timestamp timestamp = Timestamp.newBuilder().setSeconds(dateTime.getMillis() / 1000) - .setNanos((int) ((dateTime.getMillis() % 1000) * 1000000)).build(); - - Metric metric = doubleGaugeMetric(timestamp); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - metric.writeTo(out); - - InputRow row = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())).get(0); - Assert.assertEquals(dateTime.getMillis(), row.getTimestampFromEpoch()); + InputRow row = parser.parseBatch(ByteBuffer.wrap(metric.toByteArray())).get(0); + Assert.assertEquals(INSTANT.toEpochMilli(), row.getTimestampFromEpoch()); assertDimensionEquals(row, "name", "metric_gauge_double"); assertDimensionEquals(row, "foo_key", "foo_value"); @@ -126,22 +121,15 @@ public void testDoubleGaugeParse() throws Exception } @Test - public void testIntGaugeParse() throws Exception + public void testIntGaugeParse() { //configure parser with desc file OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, null, null, ""); - DateTime dateTime = new DateTime(2019, 07, 12, 9, 30, ISOChronology.getInstanceUTC()); - - Timestamp timestamp = Timestamp.newBuilder().setSeconds(dateTime.getMillis() / 1000) - .setNanos((int) ((dateTime.getMillis() % 1000) * 1000000)).build(); + Metric metric = intGaugeMetric(TIMESTAMP); - Metric metric = intGaugeMetric(timestamp); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - metric.writeTo(out); - - InputRow row = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())).get(0); - Assert.assertEquals(dateTime.getMillis(), row.getTimestampFromEpoch()); + InputRow row = parser.parseBatch(ByteBuffer.wrap(metric.toByteArray())).get(0); + Assert.assertEquals(INSTANT.toEpochMilli(), row.getTimestampFromEpoch()); assertDimensionEquals(row, "name", "metric_gauge_int64"); assertDimensionEquals(row, "foo_key", "foo_value"); @@ -150,85 +138,64 @@ public void testIntGaugeParse() throws Exception } @Test - public void testSummaryParse() throws Exception + public void testSummaryParse() { //configure parser with desc file OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, null, null, ""); - DateTime dateTime = new DateTime(2019, 07, 12, 9, 30, ISOChronology.getInstanceUTC()); - - Timestamp timestamp = Timestamp.newBuilder().setSeconds(dateTime.getMillis() / 1000) - .setNanos((int) ((dateTime.getMillis() % 1000) * 1000000)).build(); - - Metric metric = summaryMetric(timestamp); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - metric.writeTo(out); + Metric metric = summaryMetric(TIMESTAMP); - List rows = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())); + List rows = parser.parseBatch(ByteBuffer.wrap(metric.toByteArray())); Assert.assertEquals(2, rows.size()); InputRow row = rows.get(0); - Assert.assertEquals(dateTime.getMillis(), row.getTimestampFromEpoch()); + Assert.assertEquals(INSTANT.toEpochMilli(), row.getTimestampFromEpoch()); assertDimensionEquals(row, "name", "metric_summary-count"); assertDimensionEquals(row, "foo_key", "foo_value"); Assert.assertEquals(40, row.getMetric("value").doubleValue(), 0.0); row = rows.get(1); - Assert.assertEquals(dateTime.getMillis(), row.getTimestampFromEpoch()); + Assert.assertEquals(INSTANT.toEpochMilli(), row.getTimestampFromEpoch()); assertDimensionEquals(row, "name", "metric_summary-sum"); assertDimensionEquals(row, "foo_key", "foo_value"); Assert.assertEquals(10, row.getMetric("value").doubleValue(), 0.0); } @Test - public void testDistributionParse() throws Exception + public void testDistributionParse() { //configure parser with desc file OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, null, null, ""); - DateTime dateTime = new DateTime(2019, 07, 12, 9, 30, ISOChronology.getInstanceUTC()); + Metric metric = distributionMetric(TIMESTAMP); - Timestamp timestamp = Timestamp.newBuilder().setSeconds(dateTime.getMillis() / 1000) - .setNanos((int) ((dateTime.getMillis() % 1000) * 1000000)).build(); - - Metric metric = distributionMetric(timestamp); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - metric.writeTo(out); - - List rows = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())); + List rows = parser.parseBatch(ByteBuffer.wrap(metric.toByteArray())); Assert.assertEquals(2, rows.size()); InputRow row = rows.get(0); - Assert.assertEquals(dateTime.getMillis(), row.getTimestampFromEpoch()); + Assert.assertEquals(INSTANT.toEpochMilli(), row.getTimestampFromEpoch()); assertDimensionEquals(row, "name", "metric_distribution-count"); assertDimensionEquals(row, "foo_key", "foo_value"); Assert.assertEquals(100, row.getMetric("value").intValue()); row = rows.get(1); - Assert.assertEquals(dateTime.getMillis(), row.getTimestampFromEpoch()); + Assert.assertEquals(INSTANT.toEpochMilli(), row.getTimestampFromEpoch()); assertDimensionEquals(row, "name", "metric_distribution-sum"); assertDimensionEquals(row, "foo_key", "foo_value"); Assert.assertEquals(500, row.getMetric("value").doubleValue(), 0.0); } @Test - public void testDimensionsParseWithParseSpecDimensions() throws Exception + public void testDimensionsParseWithParseSpecDimensions() { //configure parser with desc file OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpecWithDimensions, null, null, ""); - DateTime dateTime = new DateTime(2019, 07, 12, 9, 30, ISOChronology.getInstanceUTC()); - - Timestamp timestamp = Timestamp.newBuilder().setSeconds(dateTime.getMillis() / 1000) - .setNanos((int) ((dateTime.getMillis() % 1000) * 1000000)).build(); + Metric metric = summaryMetric(TIMESTAMP); - Metric metric = summaryMetric(timestamp); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - metric.writeTo(out); - - List rows = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())); + List rows = parser.parseBatch(ByteBuffer.wrap(metric.toByteArray())); Assert.assertEquals(2, rows.size()); @@ -245,21 +212,14 @@ public void testDimensionsParseWithParseSpecDimensions() throws Exception } @Test - public void testDimensionsParseWithoutParseSpecDimensions() throws Exception + public void testDimensionsParseWithoutParseSpecDimensions() { //configure parser with desc file OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, null, null, ""); - DateTime dateTime = new DateTime(2019, 07, 12, 9, 30, ISOChronology.getInstanceUTC()); - - Timestamp timestamp = Timestamp.newBuilder().setSeconds(dateTime.getMillis() / 1000) - .setNanos((int) ((dateTime.getMillis() % 1000) * 1000000)).build(); - - Metric metric = summaryMetric(timestamp); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - metric.writeTo(out); + Metric metric = summaryMetric(TIMESTAMP); - List rows = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())); + List rows = parser.parseBatch(ByteBuffer.wrap(metric.toByteArray())); Assert.assertEquals(2, rows.size()); @@ -278,16 +238,14 @@ public void testDimensionsParseWithoutParseSpecDimensions() throws Exception } @Test - public void testMetricNameOverride() throws Exception + public void testMetricNameOverride() { //configure parser with desc file OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, "dimension_name", null, ""); Metric metric = summaryMetric(Timestamp.getDefaultInstance()); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - metric.writeTo(out); - List rows = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())); + List rows = parser.parseBatch(ByteBuffer.wrap(metric.toByteArray())); Assert.assertEquals(2, rows.size()); @@ -305,16 +263,14 @@ public void testMetricNameOverride() throws Exception } @Test - public void testDefaultPrefix() throws Exception + public void testDefaultPrefix() { //configure parser with desc file OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, null, null, null); Metric metric = summaryMetric(Timestamp.getDefaultInstance()); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - metric.writeTo(out); - List rows = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())); + List rows = parser.parseBatch(ByteBuffer.wrap(metric.toByteArray())); Assert.assertEquals(2, rows.size()); @@ -332,16 +288,14 @@ public void testDefaultPrefix() throws Exception } @Test - public void testCustomPrefix() throws Exception + public void testCustomPrefix() { //configure parser with desc file OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, null, "descriptor.", "custom."); Metric metric = summaryMetric(Timestamp.getDefaultInstance()); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - metric.writeTo(out); - List rows = parser.parseBatch(ByteBuffer.wrap(out.toByteArray())); + List rows = parser.parseBatch(ByteBuffer.wrap(metric.toByteArray())); Assert.assertEquals(2, rows.size()); From 3b4ebba9e965ced8ba538788dbf89df1d79ae378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 22 Dec 2020 11:05:28 -0800 Subject: [PATCH 023/114] improve OpenCensusProtobufInputRowParser performance (#25) - remove the need to parse timestamps into their own column - reduce the number of times we copy maps of labels - pre-size hashmaps and arrays when possible - use loops instead of streams in critical sections Combined these changes improve parsing performance by about 15% - added benchmark for reference --- .../opencensus-extensions/pom.xml | 13 ++ .../OpenCensusProtobufInputRowParser.java | 180 +++++++++--------- .../protobuf/OpenCensusBenchmark.java | 117 ++++++++++++ 3 files changed, 219 insertions(+), 91 deletions(-) create mode 100644 extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusBenchmark.java diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml index 84d0aeb91ebc..ea02f9659b74 100644 --- a/extensions-contrib/opencensus-extensions/pom.xml +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -61,6 +61,19 @@ junit test + + + org.openjdk.jmh + jmh-core + 1.27 + test + + + org.openjdk.jmh + jmh-generator-annprocess + 1.27 + test + diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java index 24a36e3a190f..087ec296c212 100644 --- a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java @@ -26,16 +26,19 @@ import com.google.common.collect.Sets; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Timestamp; +import io.opencensus.proto.metrics.v1.LabelKey; import io.opencensus.proto.metrics.v1.Metric; import io.opencensus.proto.metrics.v1.Point; import io.opencensus.proto.metrics.v1.TimeSeries; import org.apache.druid.data.input.ByteBufferInputRowParser; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.MapBasedInputRow; +import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.ParseSpec; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.common.parsers.ParseException; +import org.apache.druid.utils.CollectionUtils; import java.nio.ByteBuffer; import java.time.Instant; @@ -46,19 +49,18 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; public class OpenCensusProtobufInputRowParser implements ByteBufferInputRowParser { private static final Logger LOG = new Logger(OpenCensusProtobufInputRowParser.class); private static final String SEPARATOR = "-"; + private static final String VALUE_COLUMN = "value"; private static final String DEFAULT_METRIC_DIMENSION = "name"; - private static final String VALUE = "value"; - private static final String TIMESTAMP_COLUMN = "timestamp"; private static final String DEFAULT_RESOURCE_PREFIX = ""; + private final ParseSpec parseSpec; - private final List dimensions; + private final DimensionsSpec dimensionsSpec; private final String metricDimension; private final String metricLabelPrefix; @@ -73,12 +75,12 @@ public OpenCensusProtobufInputRowParser( ) { this.parseSpec = parseSpec; - this.dimensions = parseSpec.getDimensionsSpec().getDimensionNames(); + this.dimensionsSpec = parseSpec.getDimensionsSpec(); this.metricDimension = Strings.isNullOrEmpty(metricDimension) ? DEFAULT_METRIC_DIMENSION : metricDimension; this.metricLabelPrefix = StringUtils.nullToEmptyNonDruidDataString(metricPrefix); this.resourceLabelPrefix = resourcePrefix != null ? resourcePrefix : DEFAULT_RESOURCE_PREFIX; - LOG.info("Creating Open Census Protobuf parser with spec:" + parseSpec); + LOG.info("Creating OpenCensus Protobuf parser with spec:" + parseSpec); } @Override @@ -115,11 +117,16 @@ public OpenCensusProtobufInputRowParser withParseSpec(ParseSpec parseSpec) resourceLabelPrefix); } + + private interface LabelContext + { + void addRow(long millis, String metricName, Object value); + } + @Override public List parseBatch(ByteBuffer input) { - - Metric metric; + final Metric metric; try { metric = Metric.parseFrom(input); } @@ -128,19 +135,22 @@ public List parseBatch(ByteBuffer input) } // Process metric descriptor labels map keys. - List descriptorLabels = metric.getMetricDescriptor().getLabelKeysList().stream() - .map(s -> this.metricLabelPrefix + s.getKey()) - .collect(Collectors.toList()); + List descriptorLabels = new ArrayList<>(metric.getMetricDescriptor().getLabelKeysCount()); + for (LabelKey s : metric.getMetricDescriptor().getLabelKeysList()) { + descriptorLabels.add(this.metricLabelPrefix + s.getKey()); + } // Process resource labels map. - Map resourceLabelsMap = metric.getResource().getLabelsMap().entrySet().stream() - .collect(Collectors.toMap(entry -> this.resourceLabelPrefix + entry.getKey(), - Map.Entry::getValue)); + Map resourceLabelsMap = CollectionUtils.mapKeys( + metric.getResource().getLabelsMap(), + key -> this.resourceLabelPrefix + key + ); - final List dimensions; + final List schemaDimensions = dimensionsSpec.getDimensionNames(); - if (!this.dimensions.isEmpty()) { - dimensions = this.dimensions; + final List dimensions; + if (!schemaDimensions.isEmpty()) { + dimensions = schemaDimensions; } else { Set recordDimensions = new HashSet<>(descriptorLabels); @@ -151,95 +161,83 @@ public List parseBatch(ByteBuffer input) // map as they are derived dimensions, which get populated while parsing data for timeSeries // hence add them to recordDimensions. recordDimensions.add(metricDimension); - recordDimensions.add(VALUE); + recordDimensions.add(VALUE_COLUMN); dimensions = Lists.newArrayList( - Sets.difference(recordDimensions, parseSpec.getDimensionsSpec().getDimensionExclusions()) + Sets.difference(recordDimensions, dimensionsSpec.getDimensionExclusions()) ); } - // Flatten out the OpenCensus record into druid rows. + final int capacity = resourceLabelsMap.size() + + descriptorLabels.size() + + 2; // metric name + value columns + List rows = new ArrayList<>(); for (TimeSeries ts : metric.getTimeseriesList()) { + final LabelContext labelContext = (millis, metricName, value) -> { + // Add common resourceLabels. + Map event = new HashMap<>(capacity); + event.putAll(resourceLabelsMap); + // Add metric labels + for (int i = 0; i < metric.getMetricDescriptor().getLabelKeysCount(); i++) { + event.put(descriptorLabels.get(i), ts.getLabelValues(i).getValue()); + } + // add metric name and value + event.put(metricDimension, metricName); + event.put(VALUE_COLUMN, value); + rows.add(new MapBasedInputRow(millis, dimensions, event)); + }; - // Add common resourceLabels. - Map labels = new HashMap<>(resourceLabelsMap); - - // Add labels to record. - for (int i = 0; i < metric.getMetricDescriptor().getLabelKeysCount(); i++) { - labels.put(descriptorLabels.get(i), ts.getLabelValues(i).getValue()); - } - - // One row per timeSeries point. for (Point point : ts.getPointsList()) { - // Time in millis - final Timestamp t = point.getTimestamp(); - final long millis = Instant.ofEpochSecond(t.getSeconds(), t.getNanos()).toEpochMilli(); - labels.put(TIMESTAMP_COLUMN, millis); - - switch (point.getValueCase()) { - case DOUBLE_VALUE: - Map doubleGauge = new HashMap<>(); - doubleGauge.putAll(labels); - doubleGauge.put(metricDimension, metric.getMetricDescriptor().getName()); - doubleGauge.put(VALUE, point.getDoubleValue()); - addDerivedMetricsRow(doubleGauge, dimensions, rows); - break; - case INT64_VALUE: - HashMap intGauge = new HashMap<>(); - intGauge.putAll(labels); - intGauge.put(VALUE, point.getInt64Value()); - intGauge.put(metricDimension, metric.getMetricDescriptor().getName()); - addDerivedMetricsRow(intGauge, dimensions, rows); - break; - case SUMMARY_VALUE: - // count - Map summaryCount = new HashMap<>(); - summaryCount.putAll(labels); - summaryCount.put(metricDimension, metric.getMetricDescriptor().getName() + SEPARATOR + "count"); - summaryCount.put(VALUE, point.getSummaryValue().getCount().getValue()); - addDerivedMetricsRow(summaryCount, dimensions, rows); - - // sum - Map summarySum = new HashMap<>(); - summarySum.putAll(labels); - summarySum.put(metricDimension, metric.getMetricDescriptor().getName() + SEPARATOR + "sum"); - summarySum.put(VALUE, point.getSummaryValue().getSnapshot().getSum().getValue()); - addDerivedMetricsRow(summarySum, dimensions, rows); - - // TODO : Do we put percentiles into druid ? - break; - case DISTRIBUTION_VALUE: - // count - Map distCount = new HashMap<>(); - distCount.putAll(labels); - distCount.put(metricDimension, metric.getMetricDescriptor().getName() + SEPARATOR + "count"); - distCount.put(VALUE, point.getDistributionValue().getCount()); - addDerivedMetricsRow(distCount, dimensions, rows); - - // sum - Map distSum = new HashMap<>(); - distSum.putAll(labels); - distSum.put(metricDimension, metric.getMetricDescriptor().getName() + SEPARATOR + "sum"); - distSum.put(VALUE, point.getDistributionValue().getSum()); - addDerivedMetricsRow(distSum, dimensions, rows); - // TODO: How to handle buckets ? - break; - } + addPointRows(point, metric, labelContext); } } - return rows; } - private void addDerivedMetricsRow(Map derivedMetrics, List dimensions, - List rows) + private void addPointRows(Point point, Metric metric, LabelContext labelContext) { - rows.add(new MapBasedInputRow( - parseSpec.getTimestampSpec().extractTimestamp(derivedMetrics), - dimensions, - derivedMetrics - )); + Timestamp timestamp = point.getTimestamp(); + long millis = Instant.ofEpochSecond(timestamp.getSeconds(), timestamp.getNanos()).toEpochMilli(); + String metricName = metric.getMetricDescriptor().getName(); + + switch (point.getValueCase()) { + case DOUBLE_VALUE: + labelContext.addRow(millis, metricName, point.getDoubleValue()); + break; + + case INT64_VALUE: + labelContext.addRow(millis, metricName, point.getInt64Value()); + break; + + case SUMMARY_VALUE: + // count + labelContext.addRow( + millis, + metricName + SEPARATOR + "count", + point.getSummaryValue().getCount().getValue() + ); + // sum + labelContext.addRow( + millis, + metricName + SEPARATOR + "sum", + point.getSummaryValue().getSnapshot().getSum().getValue() + ); + break; + + // TODO : How to handle buckets and percentiles + case DISTRIBUTION_VALUE: + // count + labelContext.addRow(millis, metricName + SEPARATOR + "count", point.getDistributionValue().getCount()); + // sum + labelContext.addRow( + millis, + metricName + SEPARATOR + "sum", + point.getDistributionValue().getSum() + ); + break; + default: + } } @Override diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusBenchmark.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusBenchmark.java new file mode 100644 index 000000000000..6a0ac4a080b0 --- /dev/null +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusBenchmark.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opencensus.protobuf; + +import com.google.common.collect.Lists; +import com.google.protobuf.Timestamp; +import io.opencensus.proto.metrics.v1.LabelKey; +import io.opencensus.proto.metrics.v1.LabelValue; +import io.opencensus.proto.metrics.v1.Metric; +import io.opencensus.proto.metrics.v1.MetricDescriptor; +import io.opencensus.proto.metrics.v1.Point; +import io.opencensus.proto.metrics.v1.TimeSeries; +import io.opencensus.proto.resource.v1.Resource; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.JSONParseSpec; +import org.apache.druid.data.input.impl.TimestampSpec; +import org.apache.druid.java.util.common.parsers.JSONPathFieldSpec; +import org.apache.druid.java.util.common.parsers.JSONPathFieldType; +import org.apache.druid.java.util.common.parsers.JSONPathSpec; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.infra.Blackhole; + +import java.nio.ByteBuffer; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; + +@Fork(1) +public class OpenCensusBenchmark +{ + private static final Instant INSTANT = Instant.parse("2019-07-12T09:30:01.123Z"); + private static final Timestamp TIMESTAMP = Timestamp.newBuilder() + .setSeconds(INSTANT.getEpochSecond()) + .setNanos(INSTANT.getNano()).build(); + + private static final JSONParseSpec PARSE_SPEC = new JSONParseSpec( + new TimestampSpec("timestamp", "millis", null), + new DimensionsSpec(null, null, null), + new JSONPathSpec( + true, + Lists.newArrayList( + new JSONPathFieldSpec(JSONPathFieldType.ROOT, "name", ""), + new JSONPathFieldSpec(JSONPathFieldType.ROOT, "value", ""), + new JSONPathFieldSpec(JSONPathFieldType.ROOT, "foo_key", "") + ) + ), null, null + ); + + private static final OpenCensusProtobufInputRowParser PARSER = new OpenCensusProtobufInputRowParser(PARSE_SPEC, null, null, ""); + + private static final ByteBuffer BUFFER = ByteBuffer.wrap(createMetric().toByteArray()); + + static Metric createMetric() + { + final MetricDescriptor.Builder descriptorBuilder = MetricDescriptor.newBuilder() + .setName("io.confluent.domain/such/good/metric/wow") + .setUnit("ms") + .setType(MetricDescriptor.Type.CUMULATIVE_DOUBLE); + + + final TimeSeries.Builder tsBuilder = TimeSeries.newBuilder() + .setStartTimestamp(TIMESTAMP) + .addPoints(Point.newBuilder().setDoubleValue(42.0).build()); + for (int i = 0; i < 10; i++) { + descriptorBuilder.addLabelKeys(LabelKey.newBuilder() + .setKey("foo_key_" + i) + .build()); + tsBuilder.addLabelValues(LabelValue.newBuilder() + .setHasValue(true) + .setValue("foo_value") + .build()); + } + + final Map resourceLabels = new HashMap<>(); + for (int i = 0; i < 5; i++) { + resourceLabels.put("resoure.label_" + i, "val_" + i); + } + + return Metric.newBuilder() + .setMetricDescriptor(descriptorBuilder.build()) + .setResource( + Resource.newBuilder() + .setType("env") + .putAllLabels(resourceLabels) + .build()) + .addTimeseries(tsBuilder.build()) + .build(); + } + + @Benchmark() + public void measureSerde(Blackhole blackhole) + { + // buffer must be reset / duplicated each time to ensure each iteration reads the entire buffer from the beginning + for (InputRow row : PARSER.parseBatch(BUFFER.duplicate())) { + blackhole.consume(row); + } + } +} From d4b391a47778a44cc1efe39760a7dd01be5b1ce5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 5 Jan 2021 12:21:38 -0800 Subject: [PATCH 024/114] deprecate OpenCensusInputRowParser in favor of OpenCensusProtobufInputFormat (#26) InputRowParsers have been deprecated in favor or InputFormat. This implements the InputFormat version of the OpenCensus Protobuf parser, and deprecates the existing InputRowParser implementation. - the existing InputRowParser behavior is unchanged. - the InputFormat behaves like the InputRowParser, except for the default resource prefix which now defaults to "resource." instead of empty. - both implementations internally delegate to OpenCensusProtobufReader, which is covered by the existing InputRowParser tests. --- .../OpenCensusProtobufExtensionsModule.java | 3 +- .../OpenCensusProtobufInputFormat.java | 109 +++++++++ .../OpenCensusProtobufInputRowParser.java | 152 +------------ .../protobuf/OpenCensusProtobufReader.java | 209 ++++++++++++++++++ .../protobuf/OpenCensusInputFormatTest.java | 56 +++++ .../OpenCensusProtobufInputRowParserTest.java | 138 ++++++------ 6 files changed, 464 insertions(+), 203 deletions(-) create mode 100644 extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputFormat.java create mode 100644 extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java create mode 100644 extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusInputFormatTest.java diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufExtensionsModule.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufExtensionsModule.java index 39576e4629d0..66a58c0eb28e 100644 --- a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufExtensionsModule.java +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufExtensionsModule.java @@ -37,7 +37,8 @@ public List getJacksonModules() return Collections.singletonList( new SimpleModule("OpenCensusProtobufInputRowParserModule") .registerSubtypes( - new NamedType(OpenCensusProtobufInputRowParser.class, "opencensus-protobuf") + new NamedType(OpenCensusProtobufInputRowParser.class, "opencensus-protobuf"), + new NamedType(OpenCensusProtobufInputFormat.class, "opencensus-protobuf") ) ); } diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputFormat.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputFormat.java new file mode 100644 index 000000000000..2676a818922a --- /dev/null +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputFormat.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opencensus.protobuf; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.data.input.InputEntity; +import org.apache.druid.data.input.InputEntityReader; +import org.apache.druid.data.input.InputFormat; +import org.apache.druid.data.input.InputRowSchema; +import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.java.util.common.StringUtils; + +import java.io.File; +import java.util.Objects; + +public class OpenCensusProtobufInputFormat implements InputFormat +{ + private static final String DEFAULT_METRIC_DIMENSION = "name"; + private static final String DEFAULT_RESOURCE_PREFIX = "resource."; + + private final String metricDimension; + private final String metricLabelPrefix; + private final String resourceLabelPrefix; + + public OpenCensusProtobufInputFormat( + @JsonProperty("metricDimension") String metricDimension, + @JsonProperty("metricLabelPrefix") String metricLabelPrefix, + @JsonProperty("resourceLabelPrefix") String resourceLabelPrefix + ) + { + this.metricDimension = metricDimension != null ? metricDimension : DEFAULT_METRIC_DIMENSION; + this.metricLabelPrefix = StringUtils.nullToEmptyNonDruidDataString(metricLabelPrefix); + this.resourceLabelPrefix = resourceLabelPrefix != null ? resourceLabelPrefix : DEFAULT_RESOURCE_PREFIX; + } + + @Override + public boolean isSplittable() + { + return false; + } + + @Override + public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory) + { + return new OpenCensusProtobufReader( + inputRowSchema.getDimensionsSpec(), + (ByteEntity) source, + metricDimension, + metricLabelPrefix, + resourceLabelPrefix + ); + } + + @JsonProperty + public String getMetricDimension() + { + return metricDimension; + } + + @JsonProperty + public String getMetricLabelPrefix() + { + return metricLabelPrefix; + } + + @JsonProperty + public String getResourceLabelPrefix() + { + return resourceLabelPrefix; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (!(o instanceof OpenCensusProtobufInputFormat)) { + return false; + } + OpenCensusProtobufInputFormat that = (OpenCensusProtobufInputFormat) o; + return Objects.equals(metricDimension, that.metricDimension) + && Objects.equals(metricLabelPrefix, that.metricLabelPrefix) + && Objects.equals(resourceLabelPrefix, that.resourceLabelPrefix); + } + + @Override + public int hashCode() + { + return Objects.hash(metricDimension, metricLabelPrefix, resourceLabelPrefix); + } +} diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java index 087ec296c212..227a6ea5fb8b 100644 --- a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java @@ -22,45 +22,29 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Strings; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Timestamp; -import io.opencensus.proto.metrics.v1.LabelKey; -import io.opencensus.proto.metrics.v1.Metric; -import io.opencensus.proto.metrics.v1.Point; -import io.opencensus.proto.metrics.v1.TimeSeries; import org.apache.druid.data.input.ByteBufferInputRowParser; import org.apache.druid.data.input.InputRow; -import org.apache.druid.data.input.MapBasedInputRow; -import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.ParseSpec; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.logger.Logger; -import org.apache.druid.java.util.common.parsers.ParseException; -import org.apache.druid.utils.CollectionUtils; import java.nio.ByteBuffer; -import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Objects; -import java.util.Set; +/** + * use {@link OpenCensusProtobufInputFormat} instead + */ +@Deprecated public class OpenCensusProtobufInputRowParser implements ByteBufferInputRowParser { private static final Logger LOG = new Logger(OpenCensusProtobufInputRowParser.class); - private static final String SEPARATOR = "-"; - private static final String VALUE_COLUMN = "value"; private static final String DEFAULT_METRIC_DIMENSION = "name"; private static final String DEFAULT_RESOURCE_PREFIX = ""; private final ParseSpec parseSpec; - private final DimensionsSpec dimensionsSpec; private final String metricDimension; private final String metricLabelPrefix; @@ -75,7 +59,6 @@ public OpenCensusProtobufInputRowParser( ) { this.parseSpec = parseSpec; - this.dimensionsSpec = parseSpec.getDimensionsSpec(); this.metricDimension = Strings.isNullOrEmpty(metricDimension) ? DEFAULT_METRIC_DIMENSION : metricDimension; this.metricLabelPrefix = StringUtils.nullToEmptyNonDruidDataString(metricPrefix); this.resourceLabelPrefix = resourcePrefix != null ? resourcePrefix : DEFAULT_RESOURCE_PREFIX; @@ -117,127 +100,16 @@ public OpenCensusProtobufInputRowParser withParseSpec(ParseSpec parseSpec) resourceLabelPrefix); } - - private interface LabelContext - { - void addRow(long millis, String metricName, Object value); - } - @Override public List parseBatch(ByteBuffer input) { - final Metric metric; - try { - metric = Metric.parseFrom(input); - } - catch (InvalidProtocolBufferException e) { - throw new ParseException(e, "Protobuf message could not be parsed"); - } - - // Process metric descriptor labels map keys. - List descriptorLabels = new ArrayList<>(metric.getMetricDescriptor().getLabelKeysCount()); - for (LabelKey s : metric.getMetricDescriptor().getLabelKeysList()) { - descriptorLabels.add(this.metricLabelPrefix + s.getKey()); - } - - // Process resource labels map. - Map resourceLabelsMap = CollectionUtils.mapKeys( - metric.getResource().getLabelsMap(), - key -> this.resourceLabelPrefix + key - ); - - final List schemaDimensions = dimensionsSpec.getDimensionNames(); - - final List dimensions; - if (!schemaDimensions.isEmpty()) { - dimensions = schemaDimensions; - } else { - Set recordDimensions = new HashSet<>(descriptorLabels); - - // Add resource map key set to record dimensions. - recordDimensions.addAll(resourceLabelsMap.keySet()); - - // MetricDimension, VALUE dimensions will not be present in labelKeysList or Metric.Resource - // map as they are derived dimensions, which get populated while parsing data for timeSeries - // hence add them to recordDimensions. - recordDimensions.add(metricDimension); - recordDimensions.add(VALUE_COLUMN); - - dimensions = Lists.newArrayList( - Sets.difference(recordDimensions, dimensionsSpec.getDimensionExclusions()) - ); - } - - final int capacity = resourceLabelsMap.size() - + descriptorLabels.size() - + 2; // metric name + value columns - - List rows = new ArrayList<>(); - for (TimeSeries ts : metric.getTimeseriesList()) { - final LabelContext labelContext = (millis, metricName, value) -> { - // Add common resourceLabels. - Map event = new HashMap<>(capacity); - event.putAll(resourceLabelsMap); - // Add metric labels - for (int i = 0; i < metric.getMetricDescriptor().getLabelKeysCount(); i++) { - event.put(descriptorLabels.get(i), ts.getLabelValues(i).getValue()); - } - // add metric name and value - event.put(metricDimension, metricName); - event.put(VALUE_COLUMN, value); - rows.add(new MapBasedInputRow(millis, dimensions, event)); - }; - - for (Point point : ts.getPointsList()) { - addPointRows(point, metric, labelContext); - } - } - return rows; - } - - private void addPointRows(Point point, Metric metric, LabelContext labelContext) - { - Timestamp timestamp = point.getTimestamp(); - long millis = Instant.ofEpochSecond(timestamp.getSeconds(), timestamp.getNanos()).toEpochMilli(); - String metricName = metric.getMetricDescriptor().getName(); - - switch (point.getValueCase()) { - case DOUBLE_VALUE: - labelContext.addRow(millis, metricName, point.getDoubleValue()); - break; - - case INT64_VALUE: - labelContext.addRow(millis, metricName, point.getInt64Value()); - break; - - case SUMMARY_VALUE: - // count - labelContext.addRow( - millis, - metricName + SEPARATOR + "count", - point.getSummaryValue().getCount().getValue() - ); - // sum - labelContext.addRow( - millis, - metricName + SEPARATOR + "sum", - point.getSummaryValue().getSnapshot().getSum().getValue() - ); - break; - - // TODO : How to handle buckets and percentiles - case DISTRIBUTION_VALUE: - // count - labelContext.addRow(millis, metricName + SEPARATOR + "count", point.getDistributionValue().getCount()); - // sum - labelContext.addRow( - millis, - metricName + SEPARATOR + "sum", - point.getDistributionValue().getSum() - ); - break; - default: - } + return new OpenCensusProtobufReader( + parseSpec.getDimensionsSpec(), + new ByteEntity(input), + metricDimension, + metricLabelPrefix, + resourceLabelPrefix + ).readAsList(); } @Override diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java new file mode 100644 index 000000000000..9830defeaedb --- /dev/null +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opencensus.protobuf; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Timestamp; +import io.opencensus.proto.metrics.v1.LabelKey; +import io.opencensus.proto.metrics.v1.Metric; +import io.opencensus.proto.metrics.v1.Point; +import io.opencensus.proto.metrics.v1.TimeSeries; +import org.apache.druid.data.input.InputEntityReader; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.InputRowListPlusRawValues; +import org.apache.druid.data.input.MapBasedInputRow; +import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.java.util.common.CloseableIterators; +import org.apache.druid.java.util.common.parsers.CloseableIterator; +import org.apache.druid.java.util.common.parsers.ParseException; +import org.apache.druid.utils.CollectionUtils; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class OpenCensusProtobufReader implements InputEntityReader +{ + private static final String SEPARATOR = "-"; + private static final String VALUE_COLUMN = "value"; + + private final DimensionsSpec dimensionsSpec; + private final ByteEntity source; + private final String metricDimension; + private final String metricLabelPrefix; + private final String resourceLabelPrefix; + + public OpenCensusProtobufReader( + DimensionsSpec dimensionsSpec, + ByteEntity source, + String metricDimension, + String metricLabelPrefix, + String resourceLabelPrefix + ) + { + this.dimensionsSpec = dimensionsSpec; + this.source = source; + this.metricDimension = metricDimension; + this.metricLabelPrefix = metricLabelPrefix; + this.resourceLabelPrefix = resourceLabelPrefix; + } + + private interface LabelContext + { + void addRow(long millis, String metricName, Object value); + } + + @Override + public CloseableIterator read() + { + return CloseableIterators.withEmptyBaggage(readAsList().iterator()); + } + + List readAsList() + { + try { + return parseMetric(Metric.parseFrom(source.getBuffer())); + } + catch (InvalidProtocolBufferException e) { + throw new ParseException(e, "Protobuf message could not be parsed"); + } + } + + private List parseMetric(final Metric metric) + { + // Process metric descriptor labels map keys. + List descriptorLabels = new ArrayList<>(metric.getMetricDescriptor().getLabelKeysCount()); + for (LabelKey s : metric.getMetricDescriptor().getLabelKeysList()) { + descriptorLabels.add(this.metricLabelPrefix + s.getKey()); + } + + // Process resource labels map. + Map resourceLabelsMap = CollectionUtils.mapKeys( + metric.getResource().getLabelsMap(), + key -> this.resourceLabelPrefix + key + ); + + final List schemaDimensions = dimensionsSpec.getDimensionNames(); + + final List dimensions; + if (!schemaDimensions.isEmpty()) { + dimensions = schemaDimensions; + } else { + Set recordDimensions = new HashSet<>(descriptorLabels); + + // Add resource map key set to record dimensions. + recordDimensions.addAll(resourceLabelsMap.keySet()); + + // MetricDimension, VALUE dimensions will not be present in labelKeysList or Metric.Resource + // map as they are derived dimensions, which get populated while parsing data for timeSeries + // hence add them to recordDimensions. + recordDimensions.add(metricDimension); + recordDimensions.add(VALUE_COLUMN); + + dimensions = Lists.newArrayList( + Sets.difference(recordDimensions, dimensionsSpec.getDimensionExclusions()) + ); + } + + final int capacity = resourceLabelsMap.size() + + descriptorLabels.size() + + 2; // metric name + value columns + + List rows = new ArrayList<>(); + for (TimeSeries ts : metric.getTimeseriesList()) { + final LabelContext labelContext = (millis, metricName, value) -> { + // Add common resourceLabels. + Map event = new HashMap<>(capacity); + event.putAll(resourceLabelsMap); + // Add metric labels + for (int i = 0; i < metric.getMetricDescriptor().getLabelKeysCount(); i++) { + event.put(descriptorLabels.get(i), ts.getLabelValues(i).getValue()); + } + // add metric name and value + event.put(metricDimension, metricName); + event.put(VALUE_COLUMN, value); + rows.add(new MapBasedInputRow(millis, dimensions, event)); + }; + + for (Point point : ts.getPointsList()) { + addPointRows(point, metric, labelContext); + } + } + return rows; + } + + private void addPointRows(Point point, Metric metric, LabelContext labelContext) + { + Timestamp timestamp = point.getTimestamp(); + long millis = Instant.ofEpochSecond(timestamp.getSeconds(), timestamp.getNanos()).toEpochMilli(); + String metricName = metric.getMetricDescriptor().getName(); + + switch (point.getValueCase()) { + case DOUBLE_VALUE: + labelContext.addRow(millis, metricName, point.getDoubleValue()); + break; + + case INT64_VALUE: + labelContext.addRow(millis, metricName, point.getInt64Value()); + break; + + case SUMMARY_VALUE: + // count + labelContext.addRow( + millis, + metricName + SEPARATOR + "count", + point.getSummaryValue().getCount().getValue() + ); + // sum + labelContext.addRow( + millis, + metricName + SEPARATOR + "sum", + point.getSummaryValue().getSnapshot().getSum().getValue() + ); + break; + + // TODO : How to handle buckets and percentiles + case DISTRIBUTION_VALUE: + // count + labelContext.addRow(millis, metricName + SEPARATOR + "count", point.getDistributionValue().getCount()); + // sum + labelContext.addRow( + millis, + metricName + SEPARATOR + "sum", + point.getDistributionValue().getSum() + ); + break; + default: + } + } + + @Override + public CloseableIterator sample() + { + return read().map(row -> InputRowListPlusRawValues.of(row, ((MapBasedInputRow) row).getEvent())); + } +} diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusInputFormatTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusInputFormatTest.java new file mode 100644 index 000000000000..2ee056b67448 --- /dev/null +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusInputFormatTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opencensus.protobuf; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.druid.data.input.InputFormat; +import org.junit.Assert; +import org.junit.Test; + +public class OpenCensusInputFormatTest +{ + @Test + public void testSerde() throws Exception + { + OpenCensusProtobufInputFormat inputFormat = new OpenCensusProtobufInputFormat("metric.name", "descriptor.", "custom."); + + final ObjectMapper jsonMapper = new ObjectMapper(); + jsonMapper.registerModules(new OpenCensusProtobufExtensionsModule().getJacksonModules()); + + final OpenCensusProtobufInputFormat actual = (OpenCensusProtobufInputFormat) jsonMapper.readValue( + jsonMapper.writeValueAsString(inputFormat), + InputFormat.class + ); + Assert.assertEquals(inputFormat, actual); + Assert.assertEquals("metric.name", actual.getMetricDimension()); + Assert.assertEquals("descriptor.", actual.getMetricLabelPrefix()); + Assert.assertEquals("custom.", actual.getResourceLabelPrefix()); + } + + @Test + public void testDefaults() + { + OpenCensusProtobufInputFormat inputFormat = new OpenCensusProtobufInputFormat(null, null, null); + + Assert.assertEquals("name", inputFormat.getMetricDimension()); + Assert.assertEquals("", inputFormat.getMetricLabelPrefix()); + Assert.assertEquals("resource.", inputFormat.getResourceLabelPrefix()); + } +} diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java index 503dc754a3cc..2e0266e7dd9b 100644 --- a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java @@ -35,18 +35,16 @@ import io.opencensus.proto.metrics.v1.SummaryValue; import io.opencensus.proto.metrics.v1.TimeSeries; import io.opencensus.proto.resource.v1.Resource; -import org.apache.druid.data.input.ByteBufferInputRowParser; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.InputRowParser; import org.apache.druid.data.input.impl.JSONParseSpec; -import org.apache.druid.data.input.impl.ParseSpec; import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.parsers.JSONPathFieldSpec; import org.apache.druid.java.util.common.parsers.JSONPathFieldType; import org.apache.druid.java.util.common.parsers.JSONPathSpec; import org.junit.Assert; -import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -62,51 +60,81 @@ public class OpenCensusProtobufInputRowParserTest private static final Timestamp TIMESTAMP = Timestamp.newBuilder() .setSeconds(INSTANT.getEpochSecond()) .setNanos(INSTANT.getNano()).build(); - @Rule - public ExpectedException expectedException = ExpectedException.none(); - private ParseSpec parseSpec; + static final JSONParseSpec PARSE_SPEC = new JSONParseSpec( + new TimestampSpec("timestamp", "millis", null), + new DimensionsSpec(null, null, null), + new JSONPathSpec( + true, + Lists.newArrayList( + new JSONPathFieldSpec(JSONPathFieldType.ROOT, "name", ""), + new JSONPathFieldSpec(JSONPathFieldType.ROOT, "value", ""), + new JSONPathFieldSpec(JSONPathFieldType.ROOT, "foo_key", "") + ) + ), null, null + ); + + static final JSONParseSpec PARSE_SPEC_WITH_DIMENSIONS = new JSONParseSpec( + new TimestampSpec("timestamp", "millis", null), + new DimensionsSpec(ImmutableList.of( + new StringDimensionSchema("foo_key"), + new StringDimensionSchema("env_key") + ), null, null), + new JSONPathSpec( + true, + Lists.newArrayList( + new JSONPathFieldSpec(JSONPathFieldType.ROOT, "name", ""), + new JSONPathFieldSpec(JSONPathFieldType.ROOT, "value", ""), + new JSONPathFieldSpec(JSONPathFieldType.ROOT, "foo_key", "") + ) + ), null, null + ); - private ParseSpec parseSpecWithDimensions; + @Rule + public ExpectedException expectedException = ExpectedException.none(); - @Before - public void setUp() + @Test + public void testSerde() throws Exception { - parseSpec = new JSONParseSpec( - new TimestampSpec("timestamp", "millis", null), - new DimensionsSpec(null, null, null), - new JSONPathSpec( - true, - Lists.newArrayList( - new JSONPathFieldSpec(JSONPathFieldType.ROOT, "name", ""), - new JSONPathFieldSpec(JSONPathFieldType.ROOT, "value", ""), - new JSONPathFieldSpec(JSONPathFieldType.ROOT, "foo_key", "") - ) - ), null, null + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser( + OpenCensusProtobufInputRowParserTest.PARSE_SPEC, + "metric.name", + "descriptor.", + "custom." ); - parseSpecWithDimensions = new JSONParseSpec( - new TimestampSpec("timestamp", "millis", null), - new DimensionsSpec(ImmutableList.of( - new StringDimensionSchema("foo_key"), - new StringDimensionSchema("env_key")), null, null), - new JSONPathSpec( - true, - Lists.newArrayList( - new JSONPathFieldSpec(JSONPathFieldType.ROOT, "name", ""), - new JSONPathFieldSpec(JSONPathFieldType.ROOT, "value", ""), - new JSONPathFieldSpec(JSONPathFieldType.ROOT, "foo_key", "") - ) - ), null, null + final ObjectMapper jsonMapper = new ObjectMapper(); + jsonMapper.registerModules(new OpenCensusProtobufExtensionsModule().getJacksonModules()); + + final OpenCensusProtobufInputRowParser actual = (OpenCensusProtobufInputRowParser) jsonMapper.readValue( + jsonMapper.writeValueAsString(parser), + InputRowParser.class ); + Assert.assertEquals(parser, actual); + Assert.assertEquals("metric.name", actual.getMetricDimension()); + Assert.assertEquals("descriptor.", actual.getMetricLabelPrefix()); + Assert.assertEquals("custom.", actual.getResourceLabelPrefix()); } + @Test + public void testDefaults() + { + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser( + OpenCensusProtobufInputRowParserTest.PARSE_SPEC, + null, null, null + ); + + Assert.assertEquals("name", parser.getMetricDimension()); + Assert.assertEquals("", parser.getMetricLabelPrefix()); + Assert.assertEquals("", parser.getResourceLabelPrefix()); + } + @Test public void testDoubleGaugeParse() { //configure parser with desc file - OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, null, null, ""); + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(PARSE_SPEC, null, null, ""); Metric metric = doubleGaugeMetric(TIMESTAMP); @@ -124,7 +152,7 @@ public void testDoubleGaugeParse() public void testIntGaugeParse() { //configure parser with desc file - OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, null, null, ""); + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(PARSE_SPEC, null, null, ""); Metric metric = intGaugeMetric(TIMESTAMP); @@ -141,7 +169,7 @@ public void testIntGaugeParse() public void testSummaryParse() { //configure parser with desc file - OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, null, null, ""); + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(PARSE_SPEC, null, null, ""); Metric metric = summaryMetric(TIMESTAMP); @@ -166,7 +194,7 @@ public void testSummaryParse() public void testDistributionParse() { //configure parser with desc file - OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, null, null, ""); + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(PARSE_SPEC, null, null, ""); Metric metric = distributionMetric(TIMESTAMP); @@ -191,7 +219,7 @@ public void testDistributionParse() public void testDimensionsParseWithParseSpecDimensions() { //configure parser with desc file - OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpecWithDimensions, null, null, ""); + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(PARSE_SPEC_WITH_DIMENSIONS, null, null, ""); Metric metric = summaryMetric(TIMESTAMP); @@ -212,10 +240,10 @@ public void testDimensionsParseWithParseSpecDimensions() } @Test - public void testDimensionsParseWithoutParseSpecDimensions() + public void testDimensionsParseWithoutPARSE_SPECDimensions() { //configure parser with desc file - OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, null, null, ""); + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(PARSE_SPEC, null, null, ""); Metric metric = summaryMetric(TIMESTAMP); @@ -241,7 +269,7 @@ public void testDimensionsParseWithoutParseSpecDimensions() public void testMetricNameOverride() { //configure parser with desc file - OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, "dimension_name", null, ""); + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(PARSE_SPEC, "dimension_name", null, ""); Metric metric = summaryMetric(Timestamp.getDefaultInstance()); @@ -266,7 +294,7 @@ public void testMetricNameOverride() public void testDefaultPrefix() { //configure parser with desc file - OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, null, null, null); + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(PARSE_SPEC, null, null, null); Metric metric = summaryMetric(Timestamp.getDefaultInstance()); @@ -291,7 +319,7 @@ public void testDefaultPrefix() public void testCustomPrefix() { //configure parser with desc file - OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, null, "descriptor.", "custom."); + OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(PARSE_SPEC, null, "descriptor.", "custom."); Metric metric = summaryMetric(Timestamp.getDefaultInstance()); @@ -312,20 +340,6 @@ public void testCustomPrefix() assertDimensionEquals(row, "custom.env_key", "env_val"); } - @Test - public void testSerde() throws Exception - { - OpenCensusProtobufInputRowParser parser = new OpenCensusProtobufInputRowParser(parseSpec, "metric.name", "descriptor.", "custom."); - - final ObjectMapper jsonMapper = new ObjectMapper(); - jsonMapper.registerModules(new OpenCensusProtobufExtensionsModule().getJacksonModules()); - - Assert.assertEquals(parser, jsonMapper.readValue( - jsonMapper.writeValueAsString(parser), - ByteBufferInputRowParser.class - )); - } - private void assertDimensionEquals(InputRow row, String dimension, Object expected) { List values = row.getDimension(dimension); @@ -334,7 +348,7 @@ private void assertDimensionEquals(InputRow row, String dimension, Object expect Assert.assertEquals(expected, values.get(0)); } - private Metric doubleGaugeMetric(Timestamp timestamp) + static Metric doubleGaugeMetric(Timestamp timestamp) { return getMetric( "metric_gauge_double", @@ -347,7 +361,7 @@ private Metric doubleGaugeMetric(Timestamp timestamp) timestamp); } - private Metric intGaugeMetric(Timestamp timestamp) + static Metric intGaugeMetric(Timestamp timestamp) { return getMetric( "metric_gauge_int64", @@ -360,7 +374,7 @@ private Metric intGaugeMetric(Timestamp timestamp) timestamp); } - private Metric summaryMetric(Timestamp timestamp) + static Metric summaryMetric(Timestamp timestamp) { SummaryValue.Snapshot snapshot = SummaryValue.Snapshot.newBuilder() @@ -408,7 +422,7 @@ private Metric summaryMetric(Timestamp timestamp) timestamp); } - private Metric distributionMetric(Timestamp timestamp) + static Metric distributionMetric(Timestamp timestamp) { DistributionValue distributionValue = DistributionValue.newBuilder() .setCount(100) @@ -426,7 +440,7 @@ private Metric distributionMetric(Timestamp timestamp) timestamp); } - private Metric getMetric(String name, String description, MetricDescriptor.Type type, Point point, Timestamp timestamp) + static Metric getMetric(String name, String description, MetricDescriptor.Type type, Point point, Timestamp timestamp) { Metric dist = Metric.newBuilder() .setMetricDescriptor( From d23f01d5f848d8812a8128d38f1fb995e8e8ad26 Mon Sep 17 00:00:00 2001 From: CodingParsley Date: Fri, 5 Feb 2021 12:34:38 -0800 Subject: [PATCH 025/114] add default query context and update timeout to 30 sec --- web-console/console-config.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web-console/console-config.js b/web-console/console-config.js index 10bdddb611af..e6989c9bf205 100644 --- a/web-console/console-config.js +++ b/web-console/console-config.js @@ -19,4 +19,8 @@ window.consoleConfig = { exampleManifestsUrl: 'https://druid.apache.org/data/example-manifests-v2.tsv', /* future configs may go here */ + "defaultQueryContext": { + "priority": -1, + "timeout": 30000 + } }; From 87b5b3efb4fa04c0656db1795eb7562e75ac6fe1 Mon Sep 17 00:00:00 2001 From: Harini Rajendran Date: Wed, 24 Feb 2021 12:55:09 -0600 Subject: [PATCH 026/114] Setting default query lane from druid console. --- web-console/console-config.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web-console/console-config.js b/web-console/console-config.js index e6989c9bf205..a1a4de01e9b2 100644 --- a/web-console/console-config.js +++ b/web-console/console-config.js @@ -21,6 +21,7 @@ window.consoleConfig = { /* future configs may go here */ "defaultQueryContext": { "priority": -1, - "timeout": 30000 + "timeout": 30000, + "lane": "console" } }; From f0cb050233c2bfcdfc3901bda1e6827548d8782c Mon Sep 17 00:00:00 2001 From: Harini Rajendran Date: Thu, 22 Apr 2021 12:20:49 -0500 Subject: [PATCH 027/114] Giving more heap space for test jvm in semaphore config. --- .semaphore/semaphore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index ef4e9737a9a6..f5ecb748c284 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -78,7 +78,7 @@ blocks: value: extensions-contrib/confluent-extensions commands: &run_tests - > - MAVEN_OPTS="${MAVEN_OPTS} -Xmx800m" ${MVN} test -pl ${MAVEN_PROJECTS} + MAVEN_OPTS="${MAVEN_OPTS} -Xmx1g" ${MVN} test -pl ${MAVEN_PROJECTS} ${MAVEN_SKIP} -Dremoteresources.skip=true - name: "Server" From 8321fc98db30348e16e79f86e9c24c633769911d Mon Sep 17 00:00:00 2001 From: Harini Rajendran Date: Mon, 26 Apr 2021 22:15:10 -0500 Subject: [PATCH 028/114] update parent pom version for Confluent extensions --- extensions-contrib/confluent-extensions/pom.xml | 2 +- extensions-contrib/opencensus-extensions/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions-contrib/confluent-extensions/pom.xml b/extensions-contrib/confluent-extensions/pom.xml index ad11d670afa7..77a7aed1392c 100644 --- a/extensions-contrib/confluent-extensions/pom.xml +++ b/extensions-contrib/confluent-extensions/pom.xml @@ -17,7 +17,7 @@ druid org.apache.druid - 0.19.0 + 0.21.0 ../../pom.xml diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml index ea02f9659b74..5526fc18fc30 100644 --- a/extensions-contrib/opencensus-extensions/pom.xml +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -31,7 +31,7 @@ druid org.apache.druid - 0.19.0 + 0.21.0 ../../pom.xml From 0d3d3acd395df78b67b5bf2dc5d940261ffa2eb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 27 Apr 2021 16:15:39 -0700 Subject: [PATCH 029/114] Add Java 11 image build and remove unused MySQL images --- distribution/pom.xml | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index e40ca8c5c4c2..fe88eadb9e60 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -50,7 +50,6 @@ --clean - true @@ -587,7 +586,7 @@ tag - docker.io/apache/incubator-druid + docker.io/apache/druid latest docker/Dockerfile @@ -599,35 +598,33 @@ tag - docker.io/apache/incubator-druid + docker.io/apache/druid ${project.version} docker/Dockerfile - tag-latest-mysql + tag-latest-jdk11 build tag - docker.io/apache/incubator-druid-mysql - latest - docker/Dockerfile.mysql - ${skipDockerMysql} + docker.io/apache/druid + latest-jdk11 + docker/Dockerfile.java11 - tag-version-mysql + tag-version-jdk11 build tag - docker.io/apache/incubator-druid-mysql - ${project.version} - docker/Dockerfile.mysql - ${skipDockerMysql} + docker.io/apache/druid + ${project.version}-jdk11 + docker/Dockerfile.java11 From bc2e92ce25765707ec5958a39e34207efe17b21e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 27 Apr 2021 16:25:16 -0700 Subject: [PATCH 030/114] fix docker image build failure caused by #10506 --- distribution/docker/Dockerfile | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/distribution/docker/Dockerfile b/distribution/docker/Dockerfile index 3cda10b09b3d..9e619f704e8c 100644 --- a/distribution/docker/Dockerfile +++ b/distribution/docker/Dockerfile @@ -59,19 +59,13 @@ COPY --from=busybox /bin/busybox /busybox/busybox RUN ["/busybox/busybox", "--install", "/bin"] -COPY --from=extractor /opt /opt -COPY ./docker/druid.sh /druid.sh - RUN addgroup -S -g 1000 druid \ && adduser -S -u 1000 -D -H -h /opt/druid -s /bin/sh -g '' -G druid druid -COPY --from=bash-static /bin/bash /bin/bash -RUN chmod 755 /bin/bash - -COPY --chown=druid:druid --from=builder /opt /opt -COPY distribution/docker/druid.sh /druid.sh -COPY distribution/docker/peon.sh /peon.sh +COPY --chown=druid:druid --from=extractor /opt /opt +COPY ./docker/druid.sh /druid.sh +COPY ./docker/peon.sh /peon.sh # create necessary directories which could be mounted as volume # /opt/druid/var is used to keep individual files(e.g. log) of each Druid service From 4b7739659180067641623316d8f9c9f2b4a95b26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Tue, 27 Apr 2021 16:45:24 -0700 Subject: [PATCH 031/114] switch build to use Java 11 by default --- .semaphore/semaphore.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index f5ecb748c284..4a254a273161 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -28,7 +28,7 @@ blocks: value: "-DskipTests -Djacoco.skip=true" prologue: commands: - - sem-version java 8 + - sem-version java 11 - checkout jobs: - name: "Install" @@ -44,7 +44,7 @@ blocks: env_vars: *env_vars prologue: commands: - - sem-version java 8 + - sem-version java 11 - checkout - cache restore jobs: From d337dbebcd7853ac417d167b16c5f91ec6e7bc1b Mon Sep 17 00:00:00 2001 From: Ivan Vankovich Date: Tue, 7 Sep 2021 09:03:42 -0700 Subject: [PATCH 032/114] Fixed forbiddenapi error --- .../input/opencensus/protobuf/OpenCensusProtobufReader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java index 9830defeaedb..42b110847d9a 100644 --- a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java @@ -20,6 +20,7 @@ package org.apache.druid.data.input.opencensus.protobuf; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Timestamp; @@ -40,7 +41,6 @@ import java.time.Instant; import java.util.ArrayList; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -137,7 +137,7 @@ private List parseMetric(final Metric metric) for (TimeSeries ts : metric.getTimeseriesList()) { final LabelContext labelContext = (millis, metricName, value) -> { // Add common resourceLabels. - Map event = new HashMap<>(capacity); + Map event = Maps.newHashMapWithExpectedSize(capacity); event.putAll(resourceLabelsMap); // Add metric labels for (int i = 0; i < metric.getMetricDescriptor().getLabelKeysCount(); i++) { From 4be5076da0ca785874b2c06ebfca607c416b0401 Mon Sep 17 00:00:00 2001 From: Ivan Vankovich Date: Tue, 7 Sep 2021 18:58:33 -0700 Subject: [PATCH 033/114] Added phases before checks --- .semaphore/semaphore.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 4a254a273161..e07a9985ab35 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -50,6 +50,7 @@ blocks: jobs: - name: "animal sniffer checks" commands: + - ${MVN} test-compile ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} - ${MVN} animal-sniffer:check --fail-at-end - name: "checkstyle" @@ -62,6 +63,7 @@ blocks: - name: "forbidden api checks" commands: + - ${MVN} compile ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} - ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end - name: "pmd checks" From 042ed102c05d583548fdef35e3d551a349985e09 Mon Sep 17 00:00:00 2001 From: Ivan Vankovich Date: Mon, 13 Sep 2021 21:29:50 -0700 Subject: [PATCH 034/114] Fixed --- .semaphore/semaphore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index e07a9985ab35..5d2a466c5cac 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -63,7 +63,7 @@ blocks: - name: "forbidden api checks" commands: - - ${MVN} compile ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} + - ${MVN} test-compile ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} - ${MVN} forbiddenapis:check forbiddenapis:testCheck --fail-at-end - name: "pmd checks" From 0b08463ce3be113d6eb52f8e6aa6030dff5a90d5 Mon Sep 17 00:00:00 2001 From: Ivan Vankovich Date: Fri, 22 Oct 2021 17:19:35 -0700 Subject: [PATCH 035/114] OpenTelemetry Emitter Extension (#47) Add OpenTelemetry Emitter Extension --- distribution/pom.xml | 2 ++ pom.xml | 1 + 2 files changed, 3 insertions(+) diff --git a/distribution/pom.xml b/distribution/pom.xml index fe88eadb9e60..fa37d6840fd5 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -453,6 +453,8 @@ org.apache.druid.extensions.contrib:druid-opencensus-extensions -c io.confluent.druid.extensions:confluent-extensions + -c + org.apache.druid.extensions.contrib:opentelemetry-emitter diff --git a/pom.xml b/pom.xml index fdc4bb07a41f..dc7360112a16 100644 --- a/pom.xml +++ b/pom.xml @@ -229,6 +229,7 @@ extensions-contrib/druid-iceberg-extensions extensions-contrib/opencensus-extensions extensions-contrib/confluent-extensions + extensions-contrib/opentelemetry-emitter distribution From 357d13a5b4173575954f4372343c8f4bbe4aadd6 Mon Sep 17 00:00:00 2001 From: Ivan Vankovich Date: Tue, 7 Dec 2021 14:03:52 -0800 Subject: [PATCH 036/114] Add dependency check (#59) * Add dependency check * Fix maven-dependency-plugin errors * Add --fail-at-end flag * Fix comment --- .semaphore/semaphore.yml | 18 +++++++++++++ .../confluent-extensions/pom.xml | 25 +++++++++++++++++++ .../opencensus-extensions/pom.xml | 24 ++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 5d2a466c5cac..260f07e3c98c 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -74,6 +74,24 @@ blocks: commands: - ${MVN} spotbugs:check --fail-at-end -pl '!benchmarks' + - name: "analyze dependencies" + commands: + - > + ${MVN} ${MAVEN_SKIP} dependency:analyze -DoutputXML=true -DignoreNonCompile=true -DfailOnWarning=true --fail-at-end || { echo " + + The dependency analysis has found a dependency that is either: + 1) Used and undeclared: These are available as a transitive dependency but should be explicitly + added to the POM to ensure the dependency version. The XML to add the dependencies to the POM is + shown above. + 2) Unused and declared: These are not needed and removing them from the POM will speed up the build + and reduce the artifact size. The dependencies to remove are shown above. + If there are false positive dependency analysis warnings, they can be suppressed: + https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html#usedDependencies + https://maven.apache.org/plugins/maven-dependency-plugin/examples/exclude-dependencies-from-dependency-analysis.html + For more information, refer to: + https://maven.apache.org/plugins/maven-dependency-plugin/analyze-mojo.html + " && false; } + - name: "Confluent Extensions" env_vars: - name: MAVEN_PROJECTS diff --git a/extensions-contrib/confluent-extensions/pom.xml b/extensions-contrib/confluent-extensions/pom.xml index 77a7aed1392c..6e6878fde11f 100644 --- a/extensions-contrib/confluent-extensions/pom.xml +++ b/extensions-contrib/confluent-extensions/pom.xml @@ -34,6 +34,31 @@ ${project.parent.version} provided + + com.google.code.findbugs + jsr305 + provided + + + com.fasterxml.jackson.core + jackson-databind + provided + + + com.google.guava + guava + provided + + + com.google.inject + guice + provided + + + com.fasterxml.jackson.core + jackson-annotations + provided + junit diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml index 5526fc18fc30..3373be42a54c 100644 --- a/extensions-contrib/opencensus-extensions/pom.xml +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -55,6 +55,30 @@ ${project.parent.version} provided + + com.fasterxml.jackson.core + jackson-databind + provided + + + com.google.protobuf + protobuf-java + + + com.google.guava + guava + provided + + + com.google.inject + guice + provided + + + com.fasterxml.jackson.core + jackson-annotations + provided + junit From 22958a5c0a53db51297ab27f5333a3e743920ba4 Mon Sep 17 00:00:00 2001 From: Marcus Greer Date: Fri, 4 Feb 2022 13:28:10 -0800 Subject: [PATCH 037/114] METRICS-3663 OpenTelemetry Metrics InputFormat (#63) * An OpenTelemetry metrics extension * An InputFormat that is able to ingest metrics that are in the OpenTelemetry format * Unit tests for the InputFormat * Benchmarking Tests for the new OpenTelemetryMetricsProtobufInputFormat --- .../opentelemetry-extensions/pom.xml | 96 ++++++ ...enTelemetryMetricsProtobufInputFormat.java | 121 ++++++++ .../OpenTelemetryMetricsProtobufReader.java | 183 +++++++++++ ...OpenTelemetryProtobufExtensionsModule.java | 49 +++ .../protobuf/OpenTelemetryBenchmark.java | 136 ++++++++ .../OpenTelemetryMetricsInputFormatTest.java | 68 ++++ ...penTelemetryMetricsProtobufReaderTest.java | 292 ++++++++++++++++++ pom.xml | 1 + 8 files changed, 946 insertions(+) create mode 100644 extensions-contrib/opentelemetry-extensions/pom.xml create mode 100644 extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufInputFormat.java create mode 100644 extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java create mode 100644 extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java create mode 100644 extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java create mode 100644 extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsInputFormatTest.java create mode 100644 extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java diff --git a/extensions-contrib/opentelemetry-extensions/pom.xml b/extensions-contrib/opentelemetry-extensions/pom.xml new file mode 100644 index 000000000000..7b02d9539b68 --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/pom.xml @@ -0,0 +1,96 @@ + + + + 4.0.0 + + org.apache.druid.extensions.contrib + druid-opentelemetry-extensions + druid-opentelemetry-extensions + druid-opentelemetry-extensions + + + 0.11.0-alpha + + + + druid + org.apache.druid + 0.21.0 + ../../pom.xml + + + + com.google.protobuf + protobuf-java + + + io.opentelemetry.proto + opentelemetry-proto + ${opentelemetry.proto.version} + + + com.google.guava + guava + provided + + + com.google.inject + guice + provided + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + com.fasterxml.jackson.core + jackson-databind + provided + + + org.apache.druid + druid-core + ${project.parent.version} + provided + + + + junit + junit + test + + + + org.openjdk.jmh + jmh-core + 1.27 + test + + + org.openjdk.jmh + jmh-generator-annprocess + 1.27 + test + + + diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufInputFormat.java new file mode 100644 index 000000000000..2089c2c7d676 --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufInputFormat.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opentelemetry.protobuf; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.data.input.InputEntity; +import org.apache.druid.data.input.InputEntityReader; +import org.apache.druid.data.input.InputFormat; +import org.apache.druid.data.input.InputRowSchema; +import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.java.util.common.StringUtils; + +import java.io.File; +import java.util.Objects; + +public class OpenTelemetryMetricsProtobufInputFormat implements InputFormat +{ + private static final String DEFAULT_METRIC_DIMENSION = "metric"; + private static final String DEFAULT_VALUE_DIMENSION = "value"; + private static final String DEFAULT_RESOURCE_PREFIX = "resource."; + + private final String metricDimension; + private final String valueDimension; + private final String metricAttributePrefix; + private final String resourceAttributePrefix; + + public OpenTelemetryMetricsProtobufInputFormat( + @JsonProperty("metricDimension") String metricDimension, + @JsonProperty("valueDimension") String valueDimension, + @JsonProperty("metricAttributePrefix") String metricAttributePrefix, + @JsonProperty("resourceAttributePrefix") String resourceAttributePrefix + ) + { + this.metricDimension = metricDimension != null ? metricDimension : DEFAULT_METRIC_DIMENSION; + this.valueDimension = valueDimension != null ? valueDimension : DEFAULT_VALUE_DIMENSION; + this.metricAttributePrefix = StringUtils.nullToEmptyNonDruidDataString(metricAttributePrefix); + this.resourceAttributePrefix = resourceAttributePrefix != null ? resourceAttributePrefix : DEFAULT_RESOURCE_PREFIX; + } + + @Override + public boolean isSplittable() + { + return false; + } + + @Override + public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory) + { + return new OpenTelemetryMetricsProtobufReader( + inputRowSchema.getDimensionsSpec(), + (ByteEntity) source, + metricDimension, + valueDimension, + metricAttributePrefix, + resourceAttributePrefix + ); + } + + @JsonProperty + public String getMetricDimension() + { + return metricDimension; + } + + @JsonProperty + public String getValueDimension() + { + return valueDimension; + } + + @JsonProperty + public String getMetricAttributePrefix() + { + return metricAttributePrefix; + } + + @JsonProperty + public String getResourceAttributePrefix() + { + return resourceAttributePrefix; + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (!(o instanceof OpenTelemetryMetricsProtobufInputFormat)) { + return false; + } + OpenTelemetryMetricsProtobufInputFormat that = (OpenTelemetryMetricsProtobufInputFormat) o; + return Objects.equals(metricDimension, that.metricDimension) + && Objects.equals(valueDimension, that.valueDimension) + && Objects.equals(metricAttributePrefix, that.metricAttributePrefix) + && Objects.equals(resourceAttributePrefix, that.resourceAttributePrefix); + } + + @Override + public int hashCode() + { + return Objects.hash(metricDimension, valueDimension, metricAttributePrefix, resourceAttributePrefix); + } +} diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java new file mode 100644 index 000000000000..fe360e2eac0f --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opentelemetry.protobuf; + +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.protobuf.InvalidProtocolBufferException; +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.metrics.v1.Metric; +import io.opentelemetry.proto.metrics.v1.MetricsData; +import io.opentelemetry.proto.metrics.v1.NumberDataPoint; +import org.apache.druid.data.input.InputEntityReader; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.InputRowListPlusRawValues; +import org.apache.druid.data.input.MapBasedInputRow; +import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.java.util.common.CloseableIterators; +import org.apache.druid.java.util.common.parsers.CloseableIterator; +import org.apache.druid.java.util.common.parsers.ParseException; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +public class OpenTelemetryMetricsProtobufReader implements InputEntityReader +{ + + private final ByteEntity source; + private final String metricDimension; + private final String valueDimension; + private final String metricAttributePrefix; + private final String resourceAttributePrefix; + private final DimensionsSpec dimensionsSpec; + + public OpenTelemetryMetricsProtobufReader( + DimensionsSpec dimensionsSpec, + ByteEntity source, + String metricDimension, + String valueDimension, + String metricAttributePrefix, + String resourceAttributePrefix + ) + { + this.dimensionsSpec = dimensionsSpec; + this.source = source; + this.metricDimension = metricDimension; + this.valueDimension = valueDimension; + this.metricAttributePrefix = metricAttributePrefix; + this.resourceAttributePrefix = resourceAttributePrefix; + } + + @Override + public CloseableIterator read() + { + return CloseableIterators.withEmptyBaggage(readAsList().iterator()); + } + + List readAsList() + { + try { + return parseMetricsData(MetricsData.parseFrom(source.getBuffer())); + } + catch (InvalidProtocolBufferException e) { + throw new ParseException(e, "Protobuf message could not be parsed"); + } + } + + private List parseMetricsData(final MetricsData metricsData) + { + return metricsData.getResourceMetricsList() + .stream() + .flatMap(resourceMetrics -> { + Map resourceAttributes = resourceMetrics.getResource() + .getAttributesList() + .stream() + .collect(Collectors.toMap(kv -> resourceAttributePrefix + kv.getKey(), + kv -> getStringValue(kv.getValue()))); + return resourceMetrics.getInstrumentationLibraryMetricsList() + .stream() + .flatMap(libraryMetrics -> libraryMetrics.getMetricsList() + .stream() + .flatMap(metric -> parseMetric(metric, resourceAttributes).stream())); + }) + .collect(Collectors.toList()); + } + + private List parseMetric(Metric metric, Map resourceAttributes) + { + final List inputRows; + String metricName = metric.getName(); + switch (metric.getDataCase()) { + case SUM: { + inputRows = new ArrayList<>(metric.getSum().getDataPointsCount()); + metric.getSum() + .getDataPointsList() + .forEach(dataPoint -> inputRows.add(parseNumberDataPoint(dataPoint, resourceAttributes, metricName))); + break; + } + case GAUGE: { + inputRows = new ArrayList<>(metric.getGauge().getDataPointsCount()); + metric.getGauge() + .getDataPointsList() + .forEach(dataPoint -> inputRows.add(parseNumberDataPoint(dataPoint, resourceAttributes, metricName))); + break; + } + // TODO Support HISTOGRAM and SUMMARY metrics + default: + throw new IllegalStateException("Unexpected value: " + metric.getDataCase()); + } + return inputRows; + } + + private InputRow parseNumberDataPoint(NumberDataPoint dataPoint, + Map resourceAttributes, + String metricName) + { + + int capacity = resourceAttributes.size() + + dataPoint.getAttributesCount() + + 2; // metric name + value columns + Map event = Maps.newHashMapWithExpectedSize(capacity); + event.put(metricDimension, metricName); + + if (dataPoint.hasAsInt()) { + event.put(valueDimension, dataPoint.getAsInt()); + } else if (dataPoint.hasAsDouble()) { + event.put(valueDimension, dataPoint.getAsDouble()); + } else { + throw new IllegalStateException("Unexpected dataPoint value type. Expected Int or Double"); + } + + event.putAll(resourceAttributes); + dataPoint.getAttributesList().forEach(att -> event.put(metricAttributePrefix + att.getKey(), + getStringValue(att.getValue()))); + + return createRow(TimeUnit.NANOSECONDS.toMillis(dataPoint.getTimeUnixNano()), event); + } + + private static String getStringValue(AnyValue value) + { + if (value.getValueCase() == AnyValue.ValueCase.STRING_VALUE) { + return value.getStringValue(); + } + throw new IllegalStateException("Unexpected value: " + value.getValueCase()); + } + + InputRow createRow(long timeUnixMilli, Map event) + { + final List dimensions; + if (!dimensionsSpec.getDimensionNames().isEmpty()) { + dimensions = dimensionsSpec.getDimensionNames(); + } else { + dimensions = new ArrayList<>(Sets.difference(event.keySet(), dimensionsSpec.getDimensionExclusions())); + } + return new MapBasedInputRow(timeUnixMilli, dimensions, event); + } + + @Override + public CloseableIterator sample() + { + return read().map(row -> InputRowListPlusRawValues.of(row, ((MapBasedInputRow) row).getEvent())); + } +} diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java new file mode 100644 index 000000000000..4c027c31248c --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryProtobufExtensionsModule.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opentelemetry.protobuf; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.inject.Binder; +import org.apache.druid.initialization.DruidModule; + +import java.util.Collections; +import java.util.List; + +public class OpenTelemetryProtobufExtensionsModule implements DruidModule +{ + + @Override + public List getJacksonModules() + { + return Collections.singletonList( + new SimpleModule("OpenTelemetryProtobufInputFormat") + .registerSubtypes( + new NamedType(OpenTelemetryMetricsProtobufInputFormat.class, "opentelemetry-metrics-protobuf") + ) + ); + } + + @Override + public void configure(Binder binder) + { + } +} diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java new file mode 100644 index 000000000000..bbb77f1c1a1f --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opentelemetry.protobuf; + +import com.google.common.collect.ImmutableList; +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics; +import io.opentelemetry.proto.metrics.v1.Metric; +import io.opentelemetry.proto.metrics.v1.MetricsData; +import io.opentelemetry.proto.metrics.v1.NumberDataPoint; +import io.opentelemetry.proto.metrics.v1.ResourceMetrics; +import io.opentelemetry.proto.resource.v1.Resource; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.InputRowSchema; +import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.StringDimensionSchema; +import org.apache.druid.java.util.common.parsers.CloseableIterator; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.infra.Blackhole; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.time.Instant; +import java.util.concurrent.TimeUnit; + +@Fork(1) +@State(Scope.Benchmark) +public class OpenTelemetryBenchmark +{ + + private static ByteBuffer BUFFER; + + @Param(value = {"1", "2", "4", "8" }) + private int resourceMetricCount = 1; + + @Param(value = {"1"}) + private int instrumentationLibraryCount = 1; + + @Param(value = {"1", "2", "4", "8" }) + private int metricsCount = 1; + + @Param(value = {"1", "2", "4", "8" }) + private int dataPointCount; + + private static final long TIMESTAMP = TimeUnit.MILLISECONDS.toNanos(Instant.parse("2019-07-12T09:30:01.123Z").toEpochMilli()); + + private static final InputRowSchema ROW_SCHEMA = new InputRowSchema(null, + new DimensionsSpec(ImmutableList.of( + new StringDimensionSchema("name"), + new StringDimensionSchema("value"), + new StringDimensionSchema("foo_key")), + null, null), + null); + + private static final OpenTelemetryMetricsProtobufInputFormat INPUT_FORMAT = + new OpenTelemetryMetricsProtobufInputFormat("name", + "value", + "", + "resource."); + + private ByteBuffer createMetricBuffer() + { + MetricsData.Builder metricsData = MetricsData.newBuilder(); + for (int i = 0; i < resourceMetricCount; i++) { + ResourceMetrics.Builder resourceMetricsBuilder = metricsData.addResourceMetricsBuilder(); + Resource.Builder resourceBuilder = resourceMetricsBuilder.getResourceBuilder(); + + for (int resourceAttributeI = 0; resourceAttributeI < 5; resourceAttributeI++) { + KeyValue.Builder resourceAttributeBuilder = resourceBuilder.addAttributesBuilder(); + resourceAttributeBuilder.setKey("resource.label_key_" + resourceAttributeI); + resourceAttributeBuilder.setValue(AnyValue.newBuilder().setStringValue("resource.label_value")); + } + + for (int j = 0; j < instrumentationLibraryCount; j++) { + InstrumentationLibraryMetrics.Builder instrumentationLibraryMetricsBuilder = + resourceMetricsBuilder.addInstrumentationLibraryMetricsBuilder(); + + for (int k = 0; k < metricsCount; k++) { + Metric.Builder metricBuilder = instrumentationLibraryMetricsBuilder.addMetricsBuilder(); + metricBuilder.setName("io.confluent.domain/such/good/metric/wow"); + + for (int l = 0; l < dataPointCount; l++) { + NumberDataPoint.Builder dataPointBuilder = metricBuilder.getSumBuilder().addDataPointsBuilder(); + dataPointBuilder.setAsDouble(42.0).setTimeUnixNano(TIMESTAMP); + + for (int metricAttributeI = 0; metricAttributeI < 10; metricAttributeI++) { + KeyValue.Builder attributeBuilder = dataPointBuilder.addAttributesBuilder(); + attributeBuilder.setKey("foo_key_" + metricAttributeI); + attributeBuilder.setValue(AnyValue.newBuilder().setStringValue("foo-value")); + } + } + } + } + } + return ByteBuffer.wrap(metricsData.build().toByteArray()); + } + + @Setup + public void init() + { + BUFFER = createMetricBuffer(); + } + + @Benchmark() + public void measureSerde(Blackhole blackhole) throws IOException + { + for (CloseableIterator it = INPUT_FORMAT.createReader(ROW_SCHEMA, new ByteEntity(BUFFER), null).read(); it.hasNext(); ) { + InputRow row = it.next(); + blackhole.consume(row); + } + } +} diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsInputFormatTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsInputFormatTest.java new file mode 100644 index 000000000000..536247ab5716 --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsInputFormatTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opentelemetry.protobuf; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.druid.data.input.InputFormat; +import org.junit.Assert; +import org.junit.Test; + +public class OpenTelemetryMetricsInputFormatTest +{ + @Test + public void testSerde() throws Exception + { + OpenTelemetryMetricsProtobufInputFormat inputFormat = new OpenTelemetryMetricsProtobufInputFormat( + "metric.name", + "raw.value", + "descriptor.", + "custom." + ); + + final ObjectMapper jsonMapper = new ObjectMapper(); + jsonMapper.registerModules(new OpenTelemetryProtobufExtensionsModule().getJacksonModules()); + + final OpenTelemetryMetricsProtobufInputFormat actual = (OpenTelemetryMetricsProtobufInputFormat) jsonMapper.readValue( + jsonMapper.writeValueAsString(inputFormat), + InputFormat.class + ); + Assert.assertEquals(inputFormat, actual); + Assert.assertEquals("metric.name", actual.getMetricDimension()); + Assert.assertEquals("raw.value", actual.getValueDimension()); + Assert.assertEquals("descriptor.", actual.getMetricAttributePrefix()); + Assert.assertEquals("custom.", actual.getResourceAttributePrefix()); + } + + @Test + public void testDefaults() + { + OpenTelemetryMetricsProtobufInputFormat inputFormat = new OpenTelemetryMetricsProtobufInputFormat( + null, + null, + null, + null + ); + + Assert.assertEquals("metric", inputFormat.getMetricDimension()); + Assert.assertEquals("value", inputFormat.getValueDimension()); + Assert.assertEquals("", inputFormat.getMetricAttributePrefix()); + Assert.assertEquals("resource.", inputFormat.getResourceAttributePrefix()); + } +} diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java new file mode 100644 index 000000000000..80bb2b84017e --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java @@ -0,0 +1,292 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opentelemetry.protobuf; + +import com.google.common.collect.ImmutableList; +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.metrics.v1.Metric; +import io.opentelemetry.proto.metrics.v1.MetricsData; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.StringDimensionSchema; +import org.apache.druid.java.util.common.parsers.CloseableIterator; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class OpenTelemetryMetricsProtobufReaderTest +{ + private static final long TIMESTAMP = TimeUnit.MILLISECONDS.toNanos(Instant.parse("2019-07-12T09:30:01.123Z").toEpochMilli()); + public static final String RESOURCE_ATTRIBUTE_COUNTRY = "country"; + public static final String RESOURCE_ATTRIBUTE_VALUE_USA = "usa"; + + public static final String RESOURCE_ATTRIBUTE_ENV = "env"; + public static final String RESOURCE_ATTRIBUTE_VALUE_DEVEL = "devel"; + + public static final String INSTRUMENTATION_LIBRARY_NAME = "mock-instr-lib"; + public static final String INSTRUMENTATION_LIBRARY_VERSION = "1.0"; + + public static final String METRIC_ATTRIBUTE_COLOR = "color"; + public static final String METRIC_ATTRIBUTE_VALUE_RED = "red"; + + public static final String METRIC_ATTRIBUTE_FOO_KEY = "foo_key"; + public static final String METRIC_ATTRIBUTE_FOO_VAL = "foo_value"; + + private final MetricsData.Builder metricsDataBuilder = MetricsData.newBuilder(); + + private final Metric.Builder metricBuilder = metricsDataBuilder.addResourceMetricsBuilder() + .addInstrumentationLibraryMetricsBuilder() + .addMetricsBuilder(); + + private final DimensionsSpec dimensionsSpec = new DimensionsSpec(ImmutableList.of( + new StringDimensionSchema("descriptor." + METRIC_ATTRIBUTE_COLOR), + new StringDimensionSchema("descriptor." + METRIC_ATTRIBUTE_FOO_KEY), + new StringDimensionSchema("custom." + RESOURCE_ATTRIBUTE_ENV), + new StringDimensionSchema("custom." + RESOURCE_ATTRIBUTE_COUNTRY) + ), null, null); + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Before + public void setUp() + { + metricsDataBuilder + .getResourceMetricsBuilder(0) + .getResourceBuilder() + .addAttributes(KeyValue.newBuilder() + .setKey(RESOURCE_ATTRIBUTE_COUNTRY) + .setValue(AnyValue.newBuilder().setStringValue(RESOURCE_ATTRIBUTE_VALUE_USA))); + + metricsDataBuilder + .getResourceMetricsBuilder(0) + .getInstrumentationLibraryMetricsBuilder(0) + .getInstrumentationLibraryBuilder() + .setName(INSTRUMENTATION_LIBRARY_NAME) + .setVersion(INSTRUMENTATION_LIBRARY_VERSION); + + } + + @Test + public void testSumWithAttributes() + { + metricBuilder + .setName("example_sum") + .getSumBuilder() + .addDataPointsBuilder() + .setAsInt(6) + .setTimeUnixNano(TIMESTAMP) + .addAttributesBuilder() // test sum with attributes + .setKey(METRIC_ATTRIBUTE_COLOR) + .setValue(AnyValue.newBuilder().setStringValue(METRIC_ATTRIBUTE_VALUE_RED).build()); + + MetricsData metricsData = metricsDataBuilder.build(); + + CloseableIterator rows = new OpenTelemetryMetricsProtobufReader( + dimensionsSpec, + new ByteEntity(metricsData.toByteArray()), + "metric.name", + "raw.value", + "descriptor.", + "custom." + ).read(); + + List rowList = new ArrayList<>(); + rows.forEachRemaining(rowList::add); + Assert.assertEquals(1, rowList.size()); + + InputRow row = rowList.get(0); + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "metric.name", "example_sum"); + assertDimensionEquals(row, "custom.country", "usa"); + assertDimensionEquals(row, "descriptor.color", "red"); + assertDimensionEquals(row, "raw.value", "6"); + } + + @Test + public void testGaugeWithAttributes() + { + metricBuilder.setName("example_gauge") + .getGaugeBuilder() + .addDataPointsBuilder() + .setAsInt(6) + .setTimeUnixNano(TIMESTAMP) + .addAttributesBuilder() // test sum with attributes + .setKey(METRIC_ATTRIBUTE_COLOR) + .setValue(AnyValue.newBuilder().setStringValue(METRIC_ATTRIBUTE_VALUE_RED).build()); + + MetricsData metricsData = metricsDataBuilder.build(); + + CloseableIterator rows = new OpenTelemetryMetricsProtobufReader( + dimensionsSpec, + new ByteEntity(metricsData.toByteArray()), + "metric.name", + "raw.value", + "descriptor.", + "custom." + ).read(); + + Assert.assertTrue(rows.hasNext()); + InputRow row = rows.next(); + + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "metric.name", "example_gauge"); + assertDimensionEquals(row, "custom.country", "usa"); + assertDimensionEquals(row, "descriptor.color", "red"); + assertDimensionEquals(row, "raw.value", "6"); + } + + @Test + public void testBatchedMetricParse() + { + metricBuilder.setName("example_sum") + .getSumBuilder() + .addDataPointsBuilder() + .setAsInt(6) + .setTimeUnixNano(TIMESTAMP) + .addAttributesBuilder() // test sum with attributes + .setKey(METRIC_ATTRIBUTE_COLOR) + .setValue(AnyValue.newBuilder().setStringValue(METRIC_ATTRIBUTE_VALUE_RED).build()); + + // Create Second Metric + Metric.Builder gaugeMetricBuilder = metricsDataBuilder.addResourceMetricsBuilder() + .addInstrumentationLibraryMetricsBuilder() + .addMetricsBuilder(); + + metricsDataBuilder.getResourceMetricsBuilder(1) + .getResourceBuilder() + .addAttributes(KeyValue.newBuilder() + .setKey(RESOURCE_ATTRIBUTE_ENV) + .setValue(AnyValue.newBuilder().setStringValue(RESOURCE_ATTRIBUTE_VALUE_DEVEL)) + .build()); + + metricsDataBuilder.getResourceMetricsBuilder(1) + .getInstrumentationLibraryMetricsBuilder(0) + .getInstrumentationLibraryBuilder() + .setName(INSTRUMENTATION_LIBRARY_NAME) + .setVersion(INSTRUMENTATION_LIBRARY_VERSION); + + gaugeMetricBuilder.setName("example_gauge") + .getGaugeBuilder() + .addDataPointsBuilder() + .setAsInt(8) + .setTimeUnixNano(TIMESTAMP) + .addAttributesBuilder() // test sum with attributes + .setKey(METRIC_ATTRIBUTE_FOO_KEY) + .setValue(AnyValue.newBuilder().setStringValue(METRIC_ATTRIBUTE_FOO_VAL).build()); + + MetricsData metricsData = metricsDataBuilder.build(); + + CloseableIterator rows = new OpenTelemetryMetricsProtobufReader( + dimensionsSpec, + new ByteEntity(metricsData.toByteArray()), + "metric.name", + "raw.value", + "descriptor.", + "custom." + ).read(); + + Assert.assertTrue(rows.hasNext()); + InputRow row = rows.next(); + + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "metric.name", "example_sum"); + assertDimensionEquals(row, "custom.country", "usa"); + assertDimensionEquals(row, "descriptor.color", "red"); + assertDimensionEquals(row, "raw.value", "6"); + + Assert.assertTrue(rows.hasNext()); + row = rows.next(); + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "metric.name", "example_gauge"); + assertDimensionEquals(row, "custom.env", "devel"); + assertDimensionEquals(row, "descriptor.foo_key", "foo_value"); + assertDimensionEquals(row, "raw.value", "8"); + + } + + @Test + public void testDimensionSpecExclusions() + { + metricsDataBuilder.getResourceMetricsBuilder(0) + .getResourceBuilder() + .addAttributesBuilder() + .setKey(RESOURCE_ATTRIBUTE_ENV) + .setValue(AnyValue.newBuilder().setStringValue(RESOURCE_ATTRIBUTE_VALUE_DEVEL).build()); + + metricBuilder.setName("example_gauge") + .getGaugeBuilder() + .addDataPointsBuilder() + .setAsInt(6) + .setTimeUnixNano(TIMESTAMP) + .addAllAttributes(ImmutableList.of( + KeyValue.newBuilder() + .setKey(METRIC_ATTRIBUTE_COLOR) + .setValue(AnyValue.newBuilder().setStringValue(METRIC_ATTRIBUTE_VALUE_RED).build()).build(), + KeyValue.newBuilder() + .setKey(METRIC_ATTRIBUTE_FOO_KEY) + .setValue(AnyValue.newBuilder().setStringValue(METRIC_ATTRIBUTE_FOO_VAL).build()).build())); + + MetricsData metricsData = metricsDataBuilder.build(); + + DimensionsSpec dimensionsSpecWithExclusions = new DimensionsSpec(null, + ImmutableList.of( + "descriptor." + METRIC_ATTRIBUTE_COLOR, + "custom." + RESOURCE_ATTRIBUTE_COUNTRY + ), null); + + CloseableIterator rows = new OpenTelemetryMetricsProtobufReader( + dimensionsSpecWithExclusions, + new ByteEntity(metricsData.toByteArray()), + "metric.name", + "raw.value", + "descriptor.", + "custom." + ).read(); + + Assert.assertTrue(rows.hasNext()); + InputRow row = rows.next(); + + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "metric.name", "example_gauge"); + assertDimensionEquals(row, "raw.value", "6"); + assertDimensionEquals(row, "custom.env", "devel"); + assertDimensionEquals(row, "descriptor.foo_key", "foo_value"); + Assert.assertFalse(row.getDimensions().contains("custom.country")); + Assert.assertFalse(row.getDimensions().contains("descriptor.color")); + } + + private void assertDimensionEquals(InputRow row, String dimension, Object expected) + { + List values = row.getDimension(dimension); + Assert.assertEquals(1, values.size()); + Assert.assertEquals(expected, values.get(0)); + } + +} diff --git a/pom.xml b/pom.xml index dc7360112a16..ee53517fb6af 100644 --- a/pom.xml +++ b/pom.xml @@ -230,6 +230,7 @@ extensions-contrib/opencensus-extensions extensions-contrib/confluent-extensions extensions-contrib/opentelemetry-emitter + extensions-contrib/opentelemetry-extensions distribution From 6df767bb40c394f84ef82e7b8ad752da1749bbaf Mon Sep 17 00:00:00 2001 From: Harini Rajendran Date: Tue, 22 Feb 2022 14:48:52 -0600 Subject: [PATCH 038/114] update parent pom version for Confluent extensions --- extensions-contrib/confluent-extensions/pom.xml | 2 +- extensions-contrib/opencensus-extensions/pom.xml | 2 +- extensions-contrib/opentelemetry-extensions/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions-contrib/confluent-extensions/pom.xml b/extensions-contrib/confluent-extensions/pom.xml index 6e6878fde11f..001ebc9221bc 100644 --- a/extensions-contrib/confluent-extensions/pom.xml +++ b/extensions-contrib/confluent-extensions/pom.xml @@ -17,7 +17,7 @@ druid org.apache.druid - 0.21.0 + 0.22.1 ../../pom.xml diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml index 3373be42a54c..c7412cb5c4b9 100644 --- a/extensions-contrib/opencensus-extensions/pom.xml +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -31,7 +31,7 @@ druid org.apache.druid - 0.21.0 + 0.22.1 ../../pom.xml diff --git a/extensions-contrib/opentelemetry-extensions/pom.xml b/extensions-contrib/opentelemetry-extensions/pom.xml index 7b02d9539b68..df7a9f69fad7 100644 --- a/extensions-contrib/opentelemetry-extensions/pom.xml +++ b/extensions-contrib/opentelemetry-extensions/pom.xml @@ -34,7 +34,7 @@ druid org.apache.druid - 0.21.0 + 0.22.1 ../../pom.xml From 005253e1b805b4145ebdada99aab7a665cf94b8a Mon Sep 17 00:00:00 2001 From: Harini Rajendran Date: Wed, 23 Feb 2022 11:40:06 -0600 Subject: [PATCH 039/114] Adding getRequiredColumns() in our custom transforms. --- .../druid/transform/ExtractTenantTopicTransform.java | 11 +++++++++++ .../druid/transform/ExtractTenantTransform.java | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTopicTransform.java b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTopicTransform.java index 6dcad8460502..914d1cebc3cb 100644 --- a/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTopicTransform.java +++ b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTopicTransform.java @@ -9,7 +9,9 @@ import org.apache.druid.segment.transform.RowFunction; import org.apache.druid.segment.transform.Transform; +import java.util.HashSet; import java.util.Objects; +import java.util.Set; public class ExtractTenantTopicTransform implements Transform { @@ -53,6 +55,15 @@ public RowFunction getRowFunction() }; } + @Override + public Set getRequiredColumns() + { + Set columns = new HashSet(); + columns.add(this.name); + columns.add(this.fieldName); + return columns; + } + @Override public boolean equals(Object o) { diff --git a/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTransform.java b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTransform.java index b4b392146ae9..4b6ad09d6400 100644 --- a/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTransform.java +++ b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTransform.java @@ -9,7 +9,9 @@ import org.apache.druid.segment.transform.RowFunction; import org.apache.druid.segment.transform.Transform; +import java.util.HashSet; import java.util.Objects; +import java.util.Set; public class ExtractTenantTransform implements Transform { @@ -53,6 +55,15 @@ public RowFunction getRowFunction() }; } + @Override + public Set getRequiredColumns() + { + Set columns = new HashSet(); + columns.add(this.name); + columns.add(this.fieldName); + return columns; + } + @Override public boolean equals(Object o) { From 1045c94f8fd95bf5091960ae0789db904085fe94 Mon Sep 17 00:00:00 2001 From: Harini Rajendran Date: Wed, 23 Feb 2022 12:50:44 -0600 Subject: [PATCH 040/114] Updating shade-plugin version in opentelemetry-emitter. --- extensions-contrib/opentelemetry-emitter/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/extensions-contrib/opentelemetry-emitter/pom.xml b/extensions-contrib/opentelemetry-emitter/pom.xml index 58415e7ea3ef..276918a5622a 100644 --- a/extensions-contrib/opentelemetry-emitter/pom.xml +++ b/extensions-contrib/opentelemetry-emitter/pom.xml @@ -164,6 +164,7 @@ org.apache.maven.plugins maven-shade-plugin + 3.2.4 opentelemetry-extension From 4b0bbff986f32e1b1429ae8d5e3a64a890187b0f Mon Sep 17 00:00:00 2001 From: Harini Rajendran Date: Thu, 24 Feb 2022 15:58:44 -0600 Subject: [PATCH 041/114] Removing the unwanted maven-shade-plugin change. --- extensions-contrib/opentelemetry-emitter/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/extensions-contrib/opentelemetry-emitter/pom.xml b/extensions-contrib/opentelemetry-emitter/pom.xml index 276918a5622a..58415e7ea3ef 100644 --- a/extensions-contrib/opentelemetry-emitter/pom.xml +++ b/extensions-contrib/opentelemetry-emitter/pom.xml @@ -164,7 +164,6 @@ org.apache.maven.plugins maven-shade-plugin - 3.2.4 opentelemetry-extension From 5d0a0389edd8a7a40cd3079fe2d571a751ec3a86 Mon Sep 17 00:00:00 2001 From: Harini Rajendran Date: Fri, 25 Feb 2022 16:29:21 -0600 Subject: [PATCH 042/114] Adding JDK version to DockerFile and removing unwanted executions from main pom.xml file. (#75) --- distribution/pom.xml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index fa37d6840fd5..8a386b9875d5 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -605,30 +605,6 @@ docker/Dockerfile - - tag-latest-jdk11 - - build - tag - - - docker.io/apache/druid - latest-jdk11 - docker/Dockerfile.java11 - - - - tag-version-jdk11 - - build - tag - - - docker.io/apache/druid - ${project.version}-jdk11 - docker/Dockerfile.java11 - - From 0a44da268e527ae6d52c2d6fdb1bde5b7cc41bb7 Mon Sep 17 00:00:00 2001 From: Harini Rajendran Date: Sat, 26 Feb 2022 10:43:43 -0600 Subject: [PATCH 043/114] Passing JDK_VERSION as build args to docker build. (#76) --- distribution/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/distribution/pom.xml b/distribution/pom.xml index 8a386b9875d5..eeba70f016c6 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -50,6 +50,7 @@ --clean + 11 @@ -609,6 +610,7 @@ ${project.version} + ${docker.jdk.version} From 05907637aa0de82e83e130c5dc0f9e1e1f2bab9f Mon Sep 17 00:00:00 2001 From: Marcus Greer Date: Thu, 24 Feb 2022 18:34:02 -0800 Subject: [PATCH 044/114] Make the OpenTelemetry InputFormat More Flexible to Metric, Value and Attribute Types (#67) --- .../OpenTelemetryMetricsProtobufReader.java | 37 ++++++++++++++----- ...rg.apache.druid.initialization.DruidModule | 16 ++++++++ 2 files changed, 44 insertions(+), 9 deletions(-) create mode 100755 extensions-contrib/opentelemetry-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java index fe360e2eac0f..2a496d733824 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java @@ -37,6 +37,7 @@ import org.apache.druid.java.util.common.parsers.ParseException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -94,7 +95,7 @@ private List parseMetricsData(final MetricsData metricsData) .getAttributesList() .stream() .collect(Collectors.toMap(kv -> resourceAttributePrefix + kv.getKey(), - kv -> getStringValue(kv.getValue()))); + kv -> parseAnyValue(kv.getValue()))); return resourceMetrics.getInstrumentationLibraryMetricsList() .stream() .flatMap(libraryMetrics -> libraryMetrics.getMetricsList() @@ -124,6 +125,11 @@ private List parseMetric(Metric metric, Map resourceAt break; } // TODO Support HISTOGRAM and SUMMARY metrics + case HISTOGRAM: + case SUMMARY: { + inputRows = Collections.emptyList(); + break; + } default: throw new IllegalStateException("Unexpected value: " + metric.getDataCase()); } @@ -143,25 +149,38 @@ private InputRow parseNumberDataPoint(NumberDataPoint dataPoint, if (dataPoint.hasAsInt()) { event.put(valueDimension, dataPoint.getAsInt()); - } else if (dataPoint.hasAsDouble()) { - event.put(valueDimension, dataPoint.getAsDouble()); } else { - throw new IllegalStateException("Unexpected dataPoint value type. Expected Int or Double"); + event.put(valueDimension, dataPoint.getAsDouble()); } event.putAll(resourceAttributes); dataPoint.getAttributesList().forEach(att -> event.put(metricAttributePrefix + att.getKey(), - getStringValue(att.getValue()))); + parseAnyValue(att.getValue()))); return createRow(TimeUnit.NANOSECONDS.toMillis(dataPoint.getTimeUnixNano()), event); } - private static String getStringValue(AnyValue value) + private static Object parseAnyValue(AnyValue value) { - if (value.getValueCase() == AnyValue.ValueCase.STRING_VALUE) { - return value.getStringValue(); + switch (value.getValueCase()) { + case INT_VALUE: + return value.getIntValue(); + case BOOL_VALUE: + return value.getBoolValue(); + case ARRAY_VALUE: + return value.getArrayValue(); + case BYTES_VALUE: + return value.getBytesValue(); + case DOUBLE_VALUE: + return value.getDoubleValue(); + case KVLIST_VALUE: + return value.getKvlistValue(); + case STRING_VALUE: + return value.getStringValue(); + default: + // VALUE_NOT_SET: + return ""; } - throw new IllegalStateException("Unexpected value: " + value.getValueCase()); } InputRow createRow(long timeUnixMilli, Map event) diff --git a/extensions-contrib/opentelemetry-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-contrib/opentelemetry-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule new file mode 100755 index 000000000000..54b4400fd2cf --- /dev/null +++ b/extensions-contrib/opentelemetry-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.druid.data.input.opencensus.protobuf.OpenCensusProtobufExtensionsModule \ No newline at end of file From c57532cbf73a882eb721f6d19a70545b289aadd6 Mon Sep 17 00:00:00 2001 From: Yun Fu Date: Mon, 28 Feb 2022 15:09:35 -0800 Subject: [PATCH 045/114] Hybrid OpenCensusProtobufInputFormat in opencensus-extensions (#69) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Support OpenTelemetry payloads in OpenCensusProtobufInputFormat Support reading mixed OpenTelemetry and OpenCensus topics based on Kafka version header * workaround classloader isolation Workaround classloader isolation by using method handles to get access to KafkaRecordEntity related methods and check record headers Co-authored-by: Xavier Léauté --- .../opencensus-extensions/pom.xml | 26 ++ .../apache/druid/data/input/KafkaUtils.java | 107 ++++++ .../OpenCensusProtobufInputFormat.java | 45 +++ .../druid/data/input/KafkaUtilsTest.java | 90 +++++ .../protobuf/OpenCensusInputFormatTest.java | 4 +- ...penTelemetryMetricsProtobufReaderTest.java | 335 ++++++++++++++++++ .../opentelemetry-extensions/pom.xml | 5 - pom.xml | 6 + .../firehose/IngestSegmentFirehoseTest.java | 3 +- 9 files changed, 613 insertions(+), 8 deletions(-) create mode 100644 extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/KafkaUtils.java create mode 100644 extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/KafkaUtilsTest.java create mode 100644 extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenTelemetryMetricsProtobufReaderTest.java diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml index c7412cb5c4b9..7f87b2db4c9f 100644 --- a/extensions-contrib/opencensus-extensions/pom.xml +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -55,6 +55,15 @@ ${project.parent.version} provided + + io.opentelemetry.proto + opentelemetry-proto + + + org.apache.druid.extensions.contrib + druid-opentelemetry-extensions + ${project.parent.version} + com.fasterxml.jackson.core jackson-databind @@ -74,6 +83,11 @@ guice provided + + com.google.code.findbugs + jsr305 + provided + com.fasterxml.jackson.core jackson-annotations @@ -85,6 +99,18 @@ junit test + + org.apache.druid.extensions + druid-kafka-indexing-service + ${project.parent.version} + test + + + org.apache.kafka + kafka-clients + ${apache.kafka.version} + test + org.openjdk.jmh diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/KafkaUtils.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/KafkaUtils.java new file mode 100644 index 000000000000..f12589e5adbe --- /dev/null +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/KafkaUtils.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.util.Objects; + +public class KafkaUtils +{ + /** + * Creates a MethodHandle that – when invoked on a KafkaRecordEntity - returns the given header value + * for the underlying KafkaRecordEntity + * + * The method handle is roughly equivalent to the following function + * + * (KafkaRecordEntity input) -> { + * Header h = input.getRecord().headers().lastHeader(header) + * if (h != null) { + * return h.value(); + * } else { + * return null; + * } + * } + * + * Since KafkaRecordEntity only exists in the kafka-indexing-service plugin classloader, + * we need to look up the relevant classes in the classloader where the InputEntity was instantiated. + * + * The handle returned by this method should be cached for the classloader it was invoked with. + * + * If the lookup fails for whatever reason, the method handle will always return null; + * + * @param classLoader the kafka-indexing-service classloader + * @param header the header value to look up + * @return a MethodHandle + */ + public static MethodHandle lookupGetHeaderMethod(ClassLoader classLoader, String header) + { + try { + Class entityType = Class.forName("org.apache.druid.data.input.kafka.KafkaRecordEntity", true, classLoader); + Class recordType = Class.forName("org.apache.kafka.clients.consumer.ConsumerRecord", true, classLoader); + Class headersType = Class.forName("org.apache.kafka.common.header.Headers", true, classLoader); + Class headerType = Class.forName("org.apache.kafka.common.header.Header", true, classLoader); + + final MethodHandles.Lookup lookup = MethodHandles.lookup(); + MethodHandle nonNullTest = lookup.findStatic(Objects.class, "nonNull", + MethodType.methodType(boolean.class, Object.class) + ).asType(MethodType.methodType(boolean.class, headerType)); + + final MethodHandle getRecordMethod = lookup.findVirtual( + entityType, + "getRecord", + MethodType.methodType(recordType) + ); + final MethodHandle headersMethod = lookup.findVirtual(recordType, "headers", MethodType.methodType(headersType)); + final MethodHandle lastHeaderMethod = lookup.findVirtual( + headersType, + "lastHeader", + MethodType.methodType(headerType, String.class) + ); + final MethodHandle valueMethod = lookup.findVirtual(headerType, "value", MethodType.methodType(byte[].class)); + + return MethodHandles.filterReturnValue( + MethodHandles.filterReturnValue( + MethodHandles.filterReturnValue(getRecordMethod, headersMethod), + MethodHandles.insertArguments(lastHeaderMethod, 1, header) + ), + // return null byte array if header is not present + MethodHandles.guardWithTest( + nonNullTest, + valueMethod, + // match valueMethod signature by dropping the header instance argument + MethodHandles.dropArguments(MethodHandles.constant(byte[].class, null), 0, headerType) + ) + ); + } + catch (ReflectiveOperationException e) { + // if lookup fails in the classloader where the InputEntity is defined, then the source may not be + // the kafka-indexing-service classloader, or method signatures did not match. + // In that case we return a method handle always returning null + return noopMethodHandle(); + } + } + + static MethodHandle noopMethodHandle() + { + return MethodHandles.dropArguments(MethodHandles.constant(byte[].class, null), 0, InputEntity.class); + } +} diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputFormat.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputFormat.java index 2676a818922a..ae919293cc1e 100644 --- a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputFormat.java +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputFormat.java @@ -24,28 +24,42 @@ import org.apache.druid.data.input.InputEntityReader; import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.InputRowSchema; +import org.apache.druid.data.input.KafkaUtils; import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryMetricsProtobufReader; import org.apache.druid.java.util.common.StringUtils; +import javax.annotation.Nullable; import java.io.File; +import java.lang.invoke.MethodHandle; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.Objects; public class OpenCensusProtobufInputFormat implements InputFormat { private static final String DEFAULT_METRIC_DIMENSION = "name"; private static final String DEFAULT_RESOURCE_PREFIX = "resource."; + private static final String DEFAULT_VALUE_DIMENSION = "value"; + private static final String VERSION_HEADER_KEY = "v"; + private static final int OPENTELEMETRY_FORMAT_VERSION = 1; private final String metricDimension; + private final String valueDimension; private final String metricLabelPrefix; private final String resourceLabelPrefix; + private volatile MethodHandle getHeaderMethod = null; + public OpenCensusProtobufInputFormat( @JsonProperty("metricDimension") String metricDimension, + @JsonProperty("valueDimension") @Nullable String valueDimension, @JsonProperty("metricLabelPrefix") String metricLabelPrefix, @JsonProperty("resourceLabelPrefix") String resourceLabelPrefix ) { this.metricDimension = metricDimension != null ? metricDimension : DEFAULT_METRIC_DIMENSION; + this.valueDimension = valueDimension != null ? valueDimension : DEFAULT_VALUE_DIMENSION; this.metricLabelPrefix = StringUtils.nullToEmptyNonDruidDataString(metricLabelPrefix); this.resourceLabelPrefix = resourceLabelPrefix != null ? resourceLabelPrefix : DEFAULT_RESOURCE_PREFIX; } @@ -59,6 +73,37 @@ public boolean isSplittable() @Override public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory) { + // assume InputEntity is always defined in a single classloader (the kafka-indexing-service classloader) + // so we only have to look it up once. To be completely correct we should cache the method based on classloader + if (getHeaderMethod == null) { + getHeaderMethod = KafkaUtils.lookupGetHeaderMethod( + source.getClass().getClassLoader(), + OpenCensusProtobufInputFormat.VERSION_HEADER_KEY + ); + } + + try { + byte[] versionHeader = (byte[]) getHeaderMethod.invoke(source); + if (versionHeader != null) { + int version = + ByteBuffer.wrap(versionHeader).order(ByteOrder.LITTLE_ENDIAN).getInt(); + if (version == OPENTELEMETRY_FORMAT_VERSION) { + return new OpenTelemetryMetricsProtobufReader( + inputRowSchema.getDimensionsSpec(), + (ByteEntity) source, + metricDimension, + valueDimension, + metricLabelPrefix, + resourceLabelPrefix + ); + } + } + } + catch (Throwable t) { + // assume input is opencensus if something went wrong + } + + return new OpenCensusProtobufReader( inputRowSchema.getDimensionsSpec(), (ByteEntity) source, diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/KafkaUtilsTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/KafkaUtilsTest.java new file mode 100644 index 000000000000..88d918ce09e9 --- /dev/null +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/KafkaUtilsTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input; + + +import com.google.common.collect.ImmutableList; +import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.data.input.kafka.KafkaRecordEntity; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeaders; +import org.apache.kafka.common.record.TimestampType; +import org.junit.Assert; +import org.junit.Test; + +import java.lang.invoke.MethodHandle; +import java.nio.ByteBuffer; + +public class KafkaUtilsTest +{ + + private static final byte[] BYTES = ByteBuffer.allocate(Integer.BYTES).putInt(42).array(); + + @Test + public void testNoopMethodHandle() throws Throwable + { + Assert.assertNull( + KafkaUtils.noopMethodHandle().invoke(new ByteEntity(new byte[]{})) + ); + } + + @Test + public void testKafkaRecordEntity() throws Throwable + { + final MethodHandle handle = KafkaUtils.lookupGetHeaderMethod(KafkaUtilsTest.class.getClassLoader(), "version"); + KafkaRecordEntity input = new KafkaRecordEntity( + new ConsumerRecord<>( + "test", + 0, + 0, + 0, + TimestampType.CREATE_TIME, + -1L, + -1, + -1, + null, + new byte[]{}, + new RecordHeaders(ImmutableList.of(new Header() + { + @Override + public String key() + { + return "version"; + } + + @Override + public byte[] value() + { + return BYTES; + } + })) + ) + ); + Assert.assertArrayEquals(BYTES, (byte[]) handle.invoke(input)); + } + + @Test(expected = ClassCastException.class) + public void testNonKafkaEntity() throws Throwable + { + final MethodHandle handle = KafkaUtils.lookupGetHeaderMethod(KafkaUtilsTest.class.getClassLoader(), "version"); + handle.invoke(new ByteEntity(new byte[]{})); + } +} diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusInputFormatTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusInputFormatTest.java index 2ee056b67448..7aeba5462612 100644 --- a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusInputFormatTest.java +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusInputFormatTest.java @@ -29,7 +29,7 @@ public class OpenCensusInputFormatTest @Test public void testSerde() throws Exception { - OpenCensusProtobufInputFormat inputFormat = new OpenCensusProtobufInputFormat("metric.name", "descriptor.", "custom."); + OpenCensusProtobufInputFormat inputFormat = new OpenCensusProtobufInputFormat("metric.name", null, "descriptor.", "custom."); final ObjectMapper jsonMapper = new ObjectMapper(); jsonMapper.registerModules(new OpenCensusProtobufExtensionsModule().getJacksonModules()); @@ -47,7 +47,7 @@ public void testSerde() throws Exception @Test public void testDefaults() { - OpenCensusProtobufInputFormat inputFormat = new OpenCensusProtobufInputFormat(null, null, null); + OpenCensusProtobufInputFormat inputFormat = new OpenCensusProtobufInputFormat(null, null, null, null); Assert.assertEquals("name", inputFormat.getMetricDimension()); Assert.assertEquals("", inputFormat.getMetricLabelPrefix()); diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenTelemetryMetricsProtobufReaderTest.java new file mode 100644 index 000000000000..c4610fbac10b --- /dev/null +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenTelemetryMetricsProtobufReaderTest.java @@ -0,0 +1,335 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opencensus.protobuf; + +import com.google.common.collect.ImmutableList; +import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.metrics.v1.Metric; +import io.opentelemetry.proto.metrics.v1.MetricsData; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.InputRowSchema; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.impl.StringDimensionSchema; +import org.apache.druid.data.input.impl.TimestampSpec; +import org.apache.druid.data.input.kafka.KafkaRecordEntity; +import org.apache.druid.java.util.common.parsers.CloseableIterator; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.Headers; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.header.internals.RecordHeaders; +import org.apache.kafka.common.record.TimestampType; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class OpenTelemetryMetricsProtobufReaderTest +{ + private static final long TIMESTAMP = TimeUnit.MILLISECONDS.toNanos(Instant.parse("2019-07-12T09:30:01.123Z").toEpochMilli()); + public static final String RESOURCE_ATTRIBUTE_COUNTRY = "country"; + public static final String RESOURCE_ATTRIBUTE_VALUE_USA = "usa"; + + public static final String RESOURCE_ATTRIBUTE_ENV = "env"; + public static final String RESOURCE_ATTRIBUTE_VALUE_DEVEL = "devel"; + + public static final String INSTRUMENTATION_LIBRARY_NAME = "mock-instr-lib"; + public static final String INSTRUMENTATION_LIBRARY_VERSION = "1.0"; + + public static final String METRIC_ATTRIBUTE_COLOR = "color"; + public static final String METRIC_ATTRIBUTE_VALUE_RED = "red"; + + public static final String METRIC_ATTRIBUTE_FOO_KEY = "foo_key"; + public static final String METRIC_ATTRIBUTE_FOO_VAL = "foo_value"; + + private final MetricsData.Builder metricsDataBuilder = MetricsData.newBuilder(); + + private final Metric.Builder metricBuilder = metricsDataBuilder.addResourceMetricsBuilder() + .addInstrumentationLibraryMetricsBuilder() + .addMetricsBuilder(); + + private final DimensionsSpec dimensionsSpec = new DimensionsSpec(ImmutableList.of( + new StringDimensionSchema("descriptor." + METRIC_ATTRIBUTE_COLOR), + new StringDimensionSchema("descriptor." + METRIC_ATTRIBUTE_FOO_KEY), + new StringDimensionSchema("custom." + RESOURCE_ATTRIBUTE_ENV), + new StringDimensionSchema("custom." + RESOURCE_ATTRIBUTE_COUNTRY) + ), null, null); + + public static final String TOPIC = "telemetry.metrics.otel"; + public static final int PARTITION = 2; + public static final long OFFSET = 13095752723L; + public static final long TS = 1643974867555L; + public static final TimestampType TSTYPE = TimestampType.CREATE_TIME; + public static final byte[] V0_HEADER_BYTES = ByteBuffer.allocate(Integer.BYTES) + .order(ByteOrder.LITTLE_ENDIAN) + .putInt(1) + .array(); + private static final Header HEADERV1 = new RecordHeader("v", V0_HEADER_BYTES); + private static final Headers HEADERS = new RecordHeaders(new Header[]{HEADERV1}); + + + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + + @Before + public void setUp() + { + metricsDataBuilder + .getResourceMetricsBuilder(0) + .getResourceBuilder() + .addAttributes(KeyValue.newBuilder() + .setKey(RESOURCE_ATTRIBUTE_COUNTRY) + .setValue(AnyValue.newBuilder().setStringValue(RESOURCE_ATTRIBUTE_VALUE_USA))); + + metricsDataBuilder + .getResourceMetricsBuilder(0) + .getInstrumentationLibraryMetricsBuilder(0) + .getInstrumentationLibraryBuilder() + .setName(INSTRUMENTATION_LIBRARY_NAME) + .setVersion(INSTRUMENTATION_LIBRARY_VERSION); + + } + + @Test + public void testSumWithAttributes() throws IOException + { + metricBuilder + .setName("example_sum") + .getSumBuilder() + .addDataPointsBuilder() + .setAsInt(6) + .setTimeUnixNano(TIMESTAMP) + .addAttributesBuilder() // test sum with attributes + .setKey(METRIC_ATTRIBUTE_COLOR) + .setValue(AnyValue.newBuilder().setStringValue(METRIC_ATTRIBUTE_VALUE_RED).build()); + + MetricsData metricsData = metricsDataBuilder.build(); + ConsumerRecord consumerRecord = new ConsumerRecord(TOPIC, PARTITION, OFFSET, TS, TSTYPE, + -1L, -1, -1, null, metricsData.toByteArray(), HEADERS); + KafkaRecordEntity kafkaRecordEntity = new KafkaRecordEntity(consumerRecord); + OpenCensusProtobufInputFormat inputFormat = new OpenCensusProtobufInputFormat("metric.name", + null, + "descriptor.", + "custom."); + + CloseableIterator rows = inputFormat.createReader(new InputRowSchema( + new TimestampSpec("timestamp", "iso", null), + dimensionsSpec, + Collections.emptyList() + ), kafkaRecordEntity, null).read(); + + List rowList = new ArrayList<>(); + rows.forEachRemaining(rowList::add); + Assert.assertEquals(1, rowList.size()); + + InputRow row = rowList.get(0); + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "metric.name", "example_sum"); + assertDimensionEquals(row, "custom.country", "usa"); + assertDimensionEquals(row, "descriptor.color", "red"); + assertDimensionEquals(row, "value", "6"); + } + + @Test + public void testGaugeWithAttributes() throws IOException + { + metricBuilder.setName("example_gauge") + .getGaugeBuilder() + .addDataPointsBuilder() + .setAsInt(6) + .setTimeUnixNano(TIMESTAMP) + .addAttributesBuilder() // test sum with attributes + .setKey(METRIC_ATTRIBUTE_COLOR) + .setValue(AnyValue.newBuilder().setStringValue(METRIC_ATTRIBUTE_VALUE_RED).build()); + + MetricsData metricsData = metricsDataBuilder.build(); + ConsumerRecord consumerRecord = new ConsumerRecord(TOPIC, PARTITION, OFFSET, TS, TSTYPE, + -1L, -1, -1, null, metricsData.toByteArray(), HEADERS); + KafkaRecordEntity kafkaRecordEntity = new KafkaRecordEntity(consumerRecord); + OpenCensusProtobufInputFormat inputFormat = new OpenCensusProtobufInputFormat("metric.name", + null, + "descriptor.", + "custom."); + CloseableIterator rows = inputFormat.createReader(new InputRowSchema( + new TimestampSpec("timestamp", "iso", null), + dimensionsSpec, + Collections.emptyList() + ), kafkaRecordEntity, null).read(); + + Assert.assertTrue(rows.hasNext()); + InputRow row = rows.next(); + + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "metric.name", "example_gauge"); + assertDimensionEquals(row, "custom.country", "usa"); + assertDimensionEquals(row, "descriptor.color", "red"); + assertDimensionEquals(row, "value", "6"); + } + + @Test + public void testBatchedMetricParse() throws IOException + { + metricBuilder.setName("example_sum") + .getSumBuilder() + .addDataPointsBuilder() + .setAsInt(6) + .setTimeUnixNano(TIMESTAMP) + .addAttributesBuilder() // test sum with attributes + .setKey(METRIC_ATTRIBUTE_COLOR) + .setValue(AnyValue.newBuilder().setStringValue(METRIC_ATTRIBUTE_VALUE_RED).build()); + + // Create Second Metric + Metric.Builder gaugeMetricBuilder = metricsDataBuilder.addResourceMetricsBuilder() + .addInstrumentationLibraryMetricsBuilder() + .addMetricsBuilder(); + + metricsDataBuilder.getResourceMetricsBuilder(1) + .getResourceBuilder() + .addAttributes(KeyValue.newBuilder() + .setKey(RESOURCE_ATTRIBUTE_ENV) + .setValue(AnyValue.newBuilder().setStringValue(RESOURCE_ATTRIBUTE_VALUE_DEVEL)) + .build()); + + metricsDataBuilder.getResourceMetricsBuilder(1) + .getInstrumentationLibraryMetricsBuilder(0) + .getInstrumentationLibraryBuilder() + .setName(INSTRUMENTATION_LIBRARY_NAME) + .setVersion(INSTRUMENTATION_LIBRARY_VERSION); + + gaugeMetricBuilder.setName("example_gauge") + .getGaugeBuilder() + .addDataPointsBuilder() + .setAsInt(8) + .setTimeUnixNano(TIMESTAMP) + .addAttributesBuilder() // test sum with attributes + .setKey(METRIC_ATTRIBUTE_FOO_KEY) + .setValue(AnyValue.newBuilder().setStringValue(METRIC_ATTRIBUTE_FOO_VAL).build()); + + MetricsData metricsData = metricsDataBuilder.build(); + ConsumerRecord consumerRecord = new ConsumerRecord(TOPIC, PARTITION, OFFSET, TS, TSTYPE, + -1L, -1, -1, null, metricsData.toByteArray(), HEADERS); + KafkaRecordEntity kafkaRecordEntity = new KafkaRecordEntity(consumerRecord); + OpenCensusProtobufInputFormat inputFormat = new OpenCensusProtobufInputFormat("metric.name", + null, + "descriptor.", + "custom."); + + CloseableIterator rows = inputFormat.createReader(new InputRowSchema( + new TimestampSpec("timestamp", "iso", null), + dimensionsSpec, + Collections.emptyList() + ), kafkaRecordEntity, null).read(); + + + Assert.assertTrue(rows.hasNext()); + InputRow row = rows.next(); + + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "metric.name", "example_sum"); + assertDimensionEquals(row, "custom.country", "usa"); + assertDimensionEquals(row, "descriptor.color", "red"); + assertDimensionEquals(row, "value", "6"); + + Assert.assertTrue(rows.hasNext()); + row = rows.next(); + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "metric.name", "example_gauge"); + assertDimensionEquals(row, "custom.env", "devel"); + assertDimensionEquals(row, "descriptor.foo_key", "foo_value"); + assertDimensionEquals(row, "value", "8"); + + } + + @Test + public void testDimensionSpecExclusions() throws IOException + { + metricsDataBuilder.getResourceMetricsBuilder(0) + .getResourceBuilder() + .addAttributesBuilder() + .setKey(RESOURCE_ATTRIBUTE_ENV) + .setValue(AnyValue.newBuilder().setStringValue(RESOURCE_ATTRIBUTE_VALUE_DEVEL).build()); + + metricBuilder.setName("example_gauge") + .getGaugeBuilder() + .addDataPointsBuilder() + .setAsInt(6) + .setTimeUnixNano(TIMESTAMP) + .addAllAttributes(ImmutableList.of( + KeyValue.newBuilder() + .setKey(METRIC_ATTRIBUTE_COLOR) + .setValue(AnyValue.newBuilder().setStringValue(METRIC_ATTRIBUTE_VALUE_RED).build()).build(), + KeyValue.newBuilder() + .setKey(METRIC_ATTRIBUTE_FOO_KEY) + .setValue(AnyValue.newBuilder().setStringValue(METRIC_ATTRIBUTE_FOO_VAL).build()).build())); + + DimensionsSpec dimensionsSpecWithExclusions = new DimensionsSpec(null, + ImmutableList.of( + "descriptor." + METRIC_ATTRIBUTE_COLOR, + "custom." + RESOURCE_ATTRIBUTE_COUNTRY + ), null); + + MetricsData metricsData = metricsDataBuilder.build(); + ConsumerRecord consumerRecord = new ConsumerRecord(TOPIC, PARTITION, OFFSET, TS, TSTYPE, + -1L, -1, -1, null, metricsData.toByteArray(), HEADERS); + KafkaRecordEntity kafkaRecordEntity = new KafkaRecordEntity(consumerRecord); + OpenCensusProtobufInputFormat inputFormat = new OpenCensusProtobufInputFormat("metric.name", + null, + "descriptor.", + "custom."); + + CloseableIterator rows = inputFormat.createReader(new InputRowSchema( + new TimestampSpec("timestamp", "iso", null), + dimensionsSpecWithExclusions, + Collections.emptyList() + ), kafkaRecordEntity, null).read(); + + + Assert.assertTrue(rows.hasNext()); + InputRow row = rows.next(); + + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "metric.name", "example_gauge"); + assertDimensionEquals(row, "value", "6"); + assertDimensionEquals(row, "custom.env", "devel"); + assertDimensionEquals(row, "descriptor.foo_key", "foo_value"); + Assert.assertFalse(row.getDimensions().contains("custom.country")); + Assert.assertFalse(row.getDimensions().contains("descriptor.color")); + } + + private void assertDimensionEquals(InputRow row, String dimension, Object expected) + { + List values = row.getDimension(dimension); + Assert.assertEquals(1, values.size()); + Assert.assertEquals(expected, values.get(0)); + } + +} diff --git a/extensions-contrib/opentelemetry-extensions/pom.xml b/extensions-contrib/opentelemetry-extensions/pom.xml index df7a9f69fad7..60f683820390 100644 --- a/extensions-contrib/opentelemetry-extensions/pom.xml +++ b/extensions-contrib/opentelemetry-extensions/pom.xml @@ -27,10 +27,6 @@ druid-opentelemetry-extensions druid-opentelemetry-extensions - - 0.11.0-alpha - - druid org.apache.druid @@ -45,7 +41,6 @@ io.opentelemetry.proto opentelemetry-proto - ${opentelemetry.proto.version} com.google.guava diff --git a/pom.xml b/pom.xml index ee53517fb6af..19113420b5b0 100644 --- a/pom.xml +++ b/pom.xml @@ -133,6 +133,7 @@ maven.org Maven Central Repository https://repo1.maven.org/maven2/ + 0.11.0-alpha 3 @@ -1305,6 +1306,11 @@ + + io.opentelemetry.proto + opentelemetry-proto + ${opentelemetry.proto.version} + diff --git a/server/src/test/java/org/apache/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java b/server/src/test/java/org/apache/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java index 44f24f879a22..4a3d827b7c25 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import org.apache.druid.collections.spatial.search.RadiusBound; +import org.apache.druid.common.config.NullHandlingTest; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.impl.DelimitedParseSpec; import org.apache.druid.data.input.impl.DimensionsSpec; @@ -66,7 +67,7 @@ /** */ @RunWith(Parameterized.class) -public class IngestSegmentFirehoseTest +public class IngestSegmentFirehoseTest extends NullHandlingTest { private static final DimensionsSpec DIMENSIONS_SPEC = new DimensionsSpec( ImmutableList.of( From 151cea5cffa1a12783c3928da90607bf66bcbcd0 Mon Sep 17 00:00:00 2001 From: Marcus Greer Date: Tue, 1 Mar 2022 11:48:43 -0800 Subject: [PATCH 046/114] Modify the OpenTelemetry ProtobufReader's Handling of Attribute Types (#77) * Only handle INT_VALUE, BOOL_VALUE, DOUBLE_VALUE and STRING_VALUE and return null otherwise * fix wrong class in the DruidModule service provider definition --- .../OpenTelemetryMetricsProtobufReader.java | 38 +++++++----- ...rg.apache.druid.initialization.DruidModule | 2 +- ...penTelemetryMetricsProtobufReaderTest.java | 61 +++++++++++++++++++ 3 files changed, 86 insertions(+), 15 deletions(-) diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java index 2a496d733824..868e1a529495 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java @@ -36,8 +36,10 @@ import org.apache.druid.java.util.common.parsers.CloseableIterator; import org.apache.druid.java.util.common.parsers.ParseException; +import javax.annotation.Nullable; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -94,8 +96,14 @@ private List parseMetricsData(final MetricsData metricsData) Map resourceAttributes = resourceMetrics.getResource() .getAttributesList() .stream() - .collect(Collectors.toMap(kv -> resourceAttributePrefix + kv.getKey(), - kv -> parseAnyValue(kv.getValue()))); + .collect(HashMap::new, + (m, kv) -> { + Object value = parseAnyValue(kv.getValue()); + if (value != null) { + m.put(resourceAttributePrefix + kv.getKey(), value); + } + }, + HashMap::putAll); return resourceMetrics.getInstrumentationLibraryMetricsList() .stream() .flatMap(libraryMetrics -> libraryMetrics.getMetricsList() @@ -142,8 +150,8 @@ private InputRow parseNumberDataPoint(NumberDataPoint dataPoint, { int capacity = resourceAttributes.size() - + dataPoint.getAttributesCount() - + 2; // metric name + value columns + + dataPoint.getAttributesCount() + + 2; // metric name + value columns Map event = Maps.newHashMapWithExpectedSize(capacity); event.put(metricDimension, metricName); @@ -154,12 +162,17 @@ private InputRow parseNumberDataPoint(NumberDataPoint dataPoint, } event.putAll(resourceAttributes); - dataPoint.getAttributesList().forEach(att -> event.put(metricAttributePrefix + att.getKey(), - parseAnyValue(att.getValue()))); + dataPoint.getAttributesList().forEach(att -> { + Object value = parseAnyValue(att.getValue()); + if (value != null) { + event.put(metricAttributePrefix + att.getKey(), value); + } + }); return createRow(TimeUnit.NANOSECONDS.toMillis(dataPoint.getTimeUnixNano()), event); } + @Nullable private static Object parseAnyValue(AnyValue value) { switch (value.getValueCase()) { @@ -167,19 +180,16 @@ private static Object parseAnyValue(AnyValue value) return value.getIntValue(); case BOOL_VALUE: return value.getBoolValue(); - case ARRAY_VALUE: - return value.getArrayValue(); - case BYTES_VALUE: - return value.getBytesValue(); case DOUBLE_VALUE: return value.getDoubleValue(); - case KVLIST_VALUE: - return value.getKvlistValue(); case STRING_VALUE: return value.getStringValue(); + + // TODO: Support KVLIST_VALUE, ARRAY_VALUE and BYTES_VALUE + default: - // VALUE_NOT_SET: - return ""; + // VALUE_NOT_SET + return null; } } diff --git a/extensions-contrib/opentelemetry-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-contrib/opentelemetry-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule index 54b4400fd2cf..46a29b6f1b78 100755 --- a/extensions-contrib/opentelemetry-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule +++ b/extensions-contrib/opentelemetry-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -13,4 +13,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -org.apache.druid.data.input.opencensus.protobuf.OpenCensusProtobufExtensionsModule \ No newline at end of file +org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryMetricsProtobufInputFormat diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java index 80bb2b84017e..8a8edafab562 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList; import io.opentelemetry.proto.common.v1.AnyValue; import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.common.v1.KeyValueList; import io.opentelemetry.proto.metrics.v1.Metric; import io.opentelemetry.proto.metrics.v1.MetricsData; import org.apache.druid.data.input.InputRow; @@ -282,6 +283,66 @@ public void testDimensionSpecExclusions() Assert.assertFalse(row.getDimensions().contains("descriptor.color")); } + @Test + public void testUnsupportedValueTypes() + { + KeyValueList kvList = KeyValueList.newBuilder() + .addValues( + KeyValue.newBuilder() + .setKey("foo") + .setValue(AnyValue.newBuilder().setStringValue("bar").build())) + .build(); + + metricsDataBuilder.getResourceMetricsBuilder(0) + .getResourceBuilder() + .addAttributesBuilder() + .setKey(RESOURCE_ATTRIBUTE_ENV) + .setValue(AnyValue.newBuilder().setKvlistValue(kvList).build()); + + metricBuilder + .setName("example_sum") + .getSumBuilder() + .addDataPointsBuilder() + .setAsInt(6) + .setTimeUnixNano(TIMESTAMP) + .addAllAttributes(ImmutableList.of( + KeyValue.newBuilder() + .setKey(METRIC_ATTRIBUTE_COLOR) + .setValue(AnyValue.newBuilder().setStringValue(METRIC_ATTRIBUTE_VALUE_RED).build()).build(), + KeyValue.newBuilder() + .setKey(METRIC_ATTRIBUTE_FOO_KEY) + .setValue(AnyValue.newBuilder().setKvlistValue(kvList).build()).build())); + + MetricsData metricsData = metricsDataBuilder.build(); + + CloseableIterator rows = new OpenTelemetryMetricsProtobufReader( + dimensionsSpec, + new ByteEntity(metricsData.toByteArray()), + "metric.name", + "raw.value", + "descriptor.", + "custom." + ).read(); + + List rowList = new ArrayList<>(); + rows.forEachRemaining(rowList::add); + Assert.assertEquals(1, rowList.size()); + + InputRow row = rowList.get(0); + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "metric.name", "example_sum"); + assertDimensionEquals(row, "custom.country", "usa"); + assertDimensionEquals(row, "descriptor.color", "red"); + + // Unsupported resource attribute type is omitted + Assert.assertEquals(0, row.getDimension("custom.env").size()); + + // Unsupported metric attribute type is omitted + Assert.assertEquals(0, row.getDimension("descriptor.foo_key").size()); + + assertDimensionEquals(row, "raw.value", "6"); + } + private void assertDimensionEquals(InputRow row, String dimension, Object expected) { List values = row.getDimension(dimension); From 8d5a6d8a58f46c68fe9261b223ee854b5356350d Mon Sep 17 00:00:00 2001 From: woshifudayun Date: Tue, 1 Mar 2022 13:15:42 -0800 Subject: [PATCH 047/114] Fixing Opencensus extension build failures. --- .../OpenTelemetryMetricsProtobufReaderTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenTelemetryMetricsProtobufReaderTest.java index c4610fbac10b..60140e2ab31d 100644 --- a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenTelemetryMetricsProtobufReaderTest.java +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenTelemetryMetricsProtobufReaderTest.java @@ -24,6 +24,7 @@ import io.opentelemetry.proto.common.v1.KeyValue; import io.opentelemetry.proto.metrics.v1.Metric; import io.opentelemetry.proto.metrics.v1.MetricsData; +import org.apache.druid.data.input.ColumnsFilter; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRowSchema; import org.apache.druid.data.input.impl.DimensionsSpec; @@ -48,7 +49,6 @@ import java.nio.ByteOrder; import java.time.Instant; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; @@ -144,7 +144,7 @@ public void testSumWithAttributes() throws IOException CloseableIterator rows = inputFormat.createReader(new InputRowSchema( new TimestampSpec("timestamp", "iso", null), dimensionsSpec, - Collections.emptyList() + ColumnsFilter.all() ), kafkaRecordEntity, null).read(); List rowList = new ArrayList<>(); @@ -182,7 +182,7 @@ public void testGaugeWithAttributes() throws IOException CloseableIterator rows = inputFormat.createReader(new InputRowSchema( new TimestampSpec("timestamp", "iso", null), dimensionsSpec, - Collections.emptyList() + ColumnsFilter.all() ), kafkaRecordEntity, null).read(); Assert.assertTrue(rows.hasNext()); @@ -246,7 +246,7 @@ public void testBatchedMetricParse() throws IOException CloseableIterator rows = inputFormat.createReader(new InputRowSchema( new TimestampSpec("timestamp", "iso", null), dimensionsSpec, - Collections.emptyList() + ColumnsFilter.all() ), kafkaRecordEntity, null).read(); @@ -309,7 +309,7 @@ public void testDimensionSpecExclusions() throws IOException CloseableIterator rows = inputFormat.createReader(new InputRowSchema( new TimestampSpec("timestamp", "iso", null), dimensionsSpecWithExclusions, - Collections.emptyList() + ColumnsFilter.all() ), kafkaRecordEntity, null).read(); From 6ed684cf1055cfb2e377d7e448634a0affb99230 Mon Sep 17 00:00:00 2001 From: Marcus Greer Date: Tue, 1 Mar 2022 14:11:11 -0800 Subject: [PATCH 048/114] fix dependency check (#79) --- extensions-contrib/opentelemetry-extensions/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/extensions-contrib/opentelemetry-extensions/pom.xml b/extensions-contrib/opentelemetry-extensions/pom.xml index 60f683820390..63aa274f699d 100644 --- a/extensions-contrib/opentelemetry-extensions/pom.xml +++ b/extensions-contrib/opentelemetry-extensions/pom.xml @@ -52,6 +52,11 @@ guice provided + + com.google.code.findbugs + jsr305 + provided + com.fasterxml.jackson.core jackson-annotations From 08400b68c5696f4fa01840a38c802b1c4d44e05b Mon Sep 17 00:00:00 2001 From: Marcus Greer Date: Thu, 3 Mar 2022 10:58:59 -0800 Subject: [PATCH 049/114] fix OpenTelemetry extension module service definition (#73) (#81) --- .../services/org.apache.druid.initialization.DruidModule | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extensions-contrib/opentelemetry-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-contrib/opentelemetry-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule index 46a29b6f1b78..b2a7d04bb635 100755 --- a/extensions-contrib/opentelemetry-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule +++ b/extensions-contrib/opentelemetry-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule @@ -13,4 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryMetricsProtobufInputFormat +org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryProtobufExtensionsModule + From 469f36e8625ff445c234e371ff5f6f11dc51f12f Mon Sep 17 00:00:00 2001 From: lokesh-lingarajan Date: Fri, 18 Mar 2022 12:53:31 -0700 Subject: [PATCH 050/114] Setting default refresh value for task view as none. (#88) As part of this we added a default parameter that can be passed for refresh widget to avoid every refresh widget getting affected. --- .../refresh-button/refresh-button.tsx | 5 +- .../views/ingestion-view/ingestion-view.tsx | 1263 +++++++++++++++++ 2 files changed, 1267 insertions(+), 1 deletion(-) create mode 100644 web-console/src/views/ingestion-view/ingestion-view.tsx diff --git a/web-console/src/components/refresh-button/refresh-button.tsx b/web-console/src/components/refresh-button/refresh-button.tsx index bbf7ed2f0714..5a5c65151d3e 100644 --- a/web-console/src/components/refresh-button/refresh-button.tsx +++ b/web-console/src/components/refresh-button/refresh-button.tsx @@ -35,13 +35,16 @@ const DELAYS: DelayLabel[] = [ export interface RefreshButtonProps { onRefresh(auto: boolean): void; localStorageKey?: LocalStorageKeys; + defaultDelay?: number; } export const RefreshButton = React.memo(function RefreshButton(props: RefreshButtonProps) { + const { onRefresh, localStorageKey, defaultDelay = 30000 } = props; + return ( ; + + resumeSupervisorId?: string; + suspendSupervisorId?: string; + resetSupervisorId?: string; + terminateSupervisorId?: string; + + showResumeAllSupervisors: boolean; + showSuspendAllSupervisors: boolean; + showTerminateAllSupervisors: boolean; + + tasksState: QueryState; + + taskFilter: Filter[]; + supervisorFilter: Filter[]; + + groupTasksBy?: 'group_id' | 'type' | 'datasource' | 'status'; + + killTaskId?: string; + + supervisorSpecDialogOpen: boolean; + taskSpecDialogOpen: boolean; + alertErrorMsg?: string; + + taskTableActionDialogId?: string; + taskTableActionDialogStatus?: string; + taskTableActionDialogActions: BasicAction[]; + supervisorTableActionDialogId?: string; + supervisorTableActionDialogActions: BasicAction[]; + hiddenTaskColumns: LocalStorageBackedVisibility; + hiddenSupervisorColumns: LocalStorageBackedVisibility; +} + +function statusToColor(status: string): string { + switch (status) { + case 'RUNNING': + return '#2167d5'; + case 'WAITING': + return '#d5631a'; + case 'PENDING': + return '#ffbf00'; + case 'SUCCESS': + return '#57d500'; + case 'FAILED': + return '#d5100a'; + case 'CANCELED': + return '#858585'; + default: + return '#0a1500'; + } +} + +function stateToColor(status: string): string { + switch (status) { + case 'UNHEALTHY_SUPERVISOR': + return '#d5100a'; + case 'UNHEALTHY_TASKS': + return '#d5100a'; + case 'PENDING': + return '#ffbf00'; + case `SUSPENDED`: + return '#ffbf00'; + case 'STOPPING': + return '#d5100a'; + case 'RUNNING': + return '#2167d5'; + default: + return '#0a1500'; + } +} + +export class IngestionView extends React.PureComponent { + private readonly supervisorQueryManager: QueryManager; + private readonly taskQueryManager: QueryManager; + static statusRanking: Record = { + RUNNING: 4, + PENDING: 3, + WAITING: 2, + SUCCESS: 1, + FAILED: 1, + }; + + static SUPERVISOR_SQL = `SELECT + "supervisor_id", "type", "source", "state", "detailed_state", "suspended" = 1 AS "suspended" +FROM sys.supervisors +ORDER BY "supervisor_id"`; + + static TASK_SQL = `WITH tasks AS (SELECT + "task_id", "group_id", "type", "datasource", "created_time", "location", "duration", "error_msg", + CASE WHEN "error_msg" = '${CANCELED_ERROR_MSG}' THEN 'CANCELED' WHEN "status" = 'RUNNING' THEN "runner_status" ELSE "status" END AS "status" + FROM sys.tasks +) +SELECT "task_id", "group_id", "type", "datasource", "created_time", "location", "duration", "error_msg", "status" +FROM tasks +ORDER BY + ( + CASE "status" + WHEN 'RUNNING' THEN 4 + WHEN 'PENDING' THEN 3 + WHEN 'WAITING' THEN 2 + ELSE 1 + END + ) DESC, + "created_time" DESC`; + + constructor(props: IngestionViewProps, context: any) { + super(props, context); + + const taskFilter: Filter[] = []; + if (props.taskId) taskFilter.push({ id: 'task_id', value: `=${props.taskId}` }); + if (props.taskGroupId) taskFilter.push({ id: 'group_id', value: `=${props.taskGroupId}` }); + if (props.datasourceId) taskFilter.push({ id: 'datasource', value: `=${props.datasourceId}` }); + + const supervisorFilter: Filter[] = []; + if (props.datasourceId) + supervisorFilter.push({ id: 'datasource', value: `=${props.datasourceId}` }); + + this.state = { + supervisorsState: QueryState.INIT, + + showResumeAllSupervisors: false, + showSuspendAllSupervisors: false, + showTerminateAllSupervisors: false, + + tasksState: QueryState.INIT, + taskFilter: taskFilter, + supervisorFilter: supervisorFilter, + + supervisorSpecDialogOpen: props.openDialog === 'supervisor', + taskSpecDialogOpen: props.openDialog === 'task', + + taskTableActionDialogActions: [], + supervisorTableActionDialogActions: [], + + hiddenTaskColumns: new LocalStorageBackedVisibility( + LocalStorageKeys.TASK_TABLE_COLUMN_SELECTION, + ), + hiddenSupervisorColumns: new LocalStorageBackedVisibility( + LocalStorageKeys.SUPERVISOR_TABLE_COLUMN_SELECTION, + ), + }; + + this.supervisorQueryManager = new QueryManager({ + processQuery: async capabilities => { + if (capabilities.hasSql()) { + return await queryDruidSql({ + query: IngestionView.SUPERVISOR_SQL, + }); + } else if (capabilities.hasOverlordAccess()) { + const supervisors = (await Api.instance.get('/druid/indexer/v1/supervisor?full')).data; + if (!Array.isArray(supervisors)) throw new Error(`Unexpected results`); + return supervisors.map((sup: any) => { + return { + supervisor_id: deepGet(sup, 'id'), + type: deepGet(sup, 'spec.tuningConfig.type'), + source: + deepGet(sup, 'spec.ioConfig.topic') || + deepGet(sup, 'spec.ioConfig.stream') || + 'n/a', + state: deepGet(sup, 'state'), + detailed_state: deepGet(sup, 'detailedState'), + suspended: Boolean(deepGet(sup, 'suspended')), + }; + }); + } else { + throw new Error(`must have SQL or overlord access`); + } + }, + onStateChange: supervisorsState => { + this.setState({ + supervisorsState, + }); + }, + }); + + this.taskQueryManager = new QueryManager({ + processQuery: async capabilities => { + if (capabilities.hasSql()) { + return await queryDruidSql({ + query: IngestionView.TASK_SQL, + }); + } else if (capabilities.hasOverlordAccess()) { + const resp = await Api.instance.get(`/druid/indexer/v1/tasks`); + return IngestionView.parseTasks(resp.data); + } else { + throw new Error(`must have SQL or overlord access`); + } + }, + onStateChange: tasksState => { + this.setState({ + tasksState, + }); + }, + }); + } + + static parseTasks = (data: any[]): TaskQueryResultRow[] => { + return data.map(d => { + return { + task_id: d.id, + group_id: d.groupId, + type: d.type, + created_time: d.createdTime, + datasource: d.dataSource, + duration: d.duration ? d.duration : 0, + error_msg: d.errorMsg, + location: d.location.host ? `${d.location.host}:${d.location.port}` : null, + status: d.statusCode === 'RUNNING' ? d.runnerStatusCode : d.statusCode, + }; + }); + }; + + private static onSecondaryPaneSizeChange(secondaryPaneSize: number) { + localStorageSet(LocalStorageKeys.INGESTION_VIEW_PANE_SIZE, String(secondaryPaneSize)); + } + + componentDidMount(): void { + const { capabilities } = this.props; + + this.supervisorQueryManager.runQuery(capabilities); + this.taskQueryManager.runQuery(capabilities); + } + + componentWillUnmount(): void { + this.supervisorQueryManager.terminate(); + this.taskQueryManager.terminate(); + } + + private readonly closeSpecDialogs = () => { + this.setState({ + supervisorSpecDialogOpen: false, + taskSpecDialogOpen: false, + }); + }; + + private readonly submitSupervisor = async (spec: JSON) => { + try { + await Api.instance.post('/druid/indexer/v1/supervisor', spec); + } catch (e) { + AppToaster.show({ + message: `Failed to submit supervisor: ${getDruidErrorMessage(e)}`, + intent: Intent.DANGER, + }); + return; + } + + AppToaster.show({ + message: 'Supervisor submitted successfully', + intent: Intent.SUCCESS, + }); + this.supervisorQueryManager.rerunLastQuery(); + }; + + private readonly submitTask = async (spec: JSON) => { + try { + await Api.instance.post('/druid/indexer/v1/task', spec); + } catch (e) { + AppToaster.show({ + message: `Failed to submit task: ${getDruidErrorMessage(e)}`, + intent: Intent.DANGER, + }); + return; + } + + AppToaster.show({ + message: 'Task submitted successfully', + intent: Intent.SUCCESS, + }); + this.taskQueryManager.rerunLastQuery(); + }; + + private getSupervisorActions( + id: string, + supervisorSuspended: boolean, + type: string, + ): BasicAction[] { + const { goToDatasource, goToStreamingDataLoader } = this.props; + + const actions: BasicAction[] = []; + if (oneOf(type, 'kafka', 'kinesis')) { + actions.push( + { + icon: IconNames.MULTI_SELECT, + title: 'Go to datasource', + onAction: () => goToDatasource(id), + }, + { + icon: IconNames.CLOUD_UPLOAD, + title: 'Open in data loader', + onAction: () => goToStreamingDataLoader(id), + }, + ); + } + actions.push( + { + icon: supervisorSuspended ? IconNames.PLAY : IconNames.PAUSE, + title: supervisorSuspended ? 'Resume' : 'Suspend', + onAction: () => + supervisorSuspended + ? this.setState({ resumeSupervisorId: id }) + : this.setState({ suspendSupervisorId: id }), + }, + { + icon: IconNames.STEP_BACKWARD, + title: 'Hard reset', + intent: Intent.DANGER, + onAction: () => this.setState({ resetSupervisorId: id }), + }, + { + icon: IconNames.CROSS, + title: 'Terminate', + intent: Intent.DANGER, + onAction: () => this.setState({ terminateSupervisorId: id }), + }, + ); + return actions; + } + + renderResumeSupervisorAction() { + const { resumeSupervisorId } = this.state; + if (!resumeSupervisorId) return; + + return ( + { + const resp = await Api.instance.post( + `/druid/indexer/v1/supervisor/${Api.encodePath(resumeSupervisorId)}/resume`, + {}, + ); + return resp.data; + }} + confirmButtonText="Resume supervisor" + successText="Supervisor has been resumed" + failText="Could not resume supervisor" + intent={Intent.PRIMARY} + onClose={() => { + this.setState({ resumeSupervisorId: undefined }); + }} + onSuccess={() => { + this.supervisorQueryManager.rerunLastQuery(); + }} + > +

{`Are you sure you want to resume supervisor '${resumeSupervisorId}'?`}

+ + ); + } + + renderSuspendSupervisorAction() { + const { suspendSupervisorId } = this.state; + if (!suspendSupervisorId) return; + + return ( + { + const resp = await Api.instance.post( + `/druid/indexer/v1/supervisor/${Api.encodePath(suspendSupervisorId)}/suspend`, + {}, + ); + return resp.data; + }} + confirmButtonText="Suspend supervisor" + successText="Supervisor has been suspended" + failText="Could not suspend supervisor" + intent={Intent.DANGER} + onClose={() => { + this.setState({ suspendSupervisorId: undefined }); + }} + onSuccess={() => { + this.supervisorQueryManager.rerunLastQuery(); + }} + > +

{`Are you sure you want to suspend supervisor '${suspendSupervisorId}'?`}

+ + ); + } + + renderResetSupervisorAction() { + const { resetSupervisorId } = this.state; + if (!resetSupervisorId) return; + + return ( + { + const resp = await Api.instance.post( + `/druid/indexer/v1/supervisor/${Api.encodePath(resetSupervisorId)}/reset`, + {}, + ); + return resp.data; + }} + confirmButtonText="Hard reset supervisor" + successText="Supervisor has been hard reset" + failText="Could not hard reset supervisor" + intent={Intent.DANGER} + onClose={() => { + this.setState({ resetSupervisorId: undefined }); + }} + onSuccess={() => { + this.supervisorQueryManager.rerunLastQuery(); + }} + warningChecks={[ + `I understand that resetting ${resetSupervisorId} will clear checkpoints and therefore lead to data loss or duplication.`, + 'I understand that this operation cannot be undone.', + ]} + > +

{`Are you sure you want to hard reset supervisor '${resetSupervisorId}'?`}

+

Hard resetting a supervisor will lead to data loss or data duplication.

+

+ The reason for using this operation is to recover from a state in which the supervisor + ceases operating due to missing offsets. +

+ + ); + } + + renderTerminateSupervisorAction() { + const { terminateSupervisorId } = this.state; + if (!terminateSupervisorId) return; + + return ( + { + const resp = await Api.instance.post( + `/druid/indexer/v1/supervisor/${Api.encodePath(terminateSupervisorId)}/terminate`, + {}, + ); + return resp.data; + }} + confirmButtonText="Terminate supervisor" + successText="Supervisor has been terminated" + failText="Could not terminate supervisor" + intent={Intent.DANGER} + onClose={() => { + this.setState({ terminateSupervisorId: undefined }); + }} + onSuccess={() => { + this.supervisorQueryManager.rerunLastQuery(); + }} + > +

{`Are you sure you want to terminate supervisor '${terminateSupervisorId}'?`}

+

This action is not reversible.

+ + ); + } + + private renderSupervisorFilterableCell(field: string) { + const { supervisorFilter } = this.state; + + return (row: { value: any }) => ( + this.setState({ supervisorFilter: filters })} + > + {row.value} + + ); + } + + private onSupervisorDetail(supervisor: SupervisorQueryResultRow) { + this.setState({ + supervisorTableActionDialogId: supervisor.supervisor_id, + supervisorTableActionDialogActions: this.getSupervisorActions( + supervisor.supervisor_id, + supervisor.suspended, + supervisor.type, + ), + }); + } + + private renderSupervisorTable() { + const { supervisorsState, hiddenSupervisorColumns, taskFilter, supervisorFilter } = this.state; + + const supervisors = supervisorsState.data || []; + return ( + { + this.setState({ + supervisorFilter: filtered, + taskFilter: + column.id === 'datasource' + ? syncFilterClauseById(taskFilter, filtered, 'datasource') + : taskFilter, + }); + }} + filterable + defaultPageSize={SMALL_TABLE_PAGE_SIZE} + pageSizeOptions={SMALL_TABLE_PAGE_SIZE_OPTIONS} + showPagination={supervisors.length > SMALL_TABLE_PAGE_SIZE} + columns={[ + { + Header: 'Datasource', + id: 'datasource', + accessor: 'supervisor_id', + width: 300, + show: hiddenSupervisorColumns.shown('Datasource'), + Cell: ({ value, original }) => ( + this.onSupervisorDetail(original)} + hoverIcon={IconNames.EDIT} + > + {value} + + ), + }, + { + Header: 'Type', + accessor: 'type', + width: 100, + Cell: this.renderSupervisorFilterableCell('type'), + show: hiddenSupervisorColumns.shown('Type'), + }, + { + Header: 'Topic/Stream', + accessor: 'source', + width: 300, + Cell: this.renderSupervisorFilterableCell('source'), + show: hiddenSupervisorColumns.shown('Topic/Stream'), + }, + { + Header: 'Status', + id: 'status', + width: 300, + accessor: 'detailed_state', + Cell: row => ( + this.setState({ supervisorFilter: filters })} + > + + ●  + {row.value} + + + ), + show: hiddenSupervisorColumns.shown('Status'), + }, + { + Header: ACTION_COLUMN_LABEL, + id: ACTION_COLUMN_ID, + accessor: 'supervisor_id', + width: ACTION_COLUMN_WIDTH, + filterable: false, + Cell: row => { + const id = row.value; + const type = row.original.type; + const supervisorSuspended = row.original.suspended; + const supervisorActions = this.getSupervisorActions(id, supervisorSuspended, type); + return ( + this.onSupervisorDetail(row.original)} + actions={supervisorActions} + /> + ); + }, + show: hiddenSupervisorColumns.shown(ACTION_COLUMN_LABEL), + }, + ]} + /> + ); + } + + private getTaskActions( + id: string, + datasource: string, + status: string, + type: string, + ): BasicAction[] { + const { goToDatasource, goToClassicBatchDataLoader } = this.props; + + const actions: BasicAction[] = []; + if (datasource && status === 'SUCCESS') { + actions.push({ + icon: IconNames.MULTI_SELECT, + title: 'Go to datasource', + onAction: () => goToDatasource(datasource), + }); + } + if (oneOf(type, 'index', 'index_parallel')) { + actions.push({ + icon: IconNames.CLOUD_UPLOAD, + title: 'Open in data loader', + onAction: () => goToClassicBatchDataLoader(id), + }); + } + if (oneOf(status, 'RUNNING', 'WAITING', 'PENDING')) { + actions.push({ + icon: IconNames.CROSS, + title: 'Kill', + intent: Intent.DANGER, + onAction: () => this.setState({ killTaskId: id }), + }); + } + return actions; + } + + renderKillTaskAction() { + const { killTaskId } = this.state; + if (!killTaskId) return; + + return ( + { + const resp = await Api.instance.post( + `/druid/indexer/v1/task/${Api.encodePath(killTaskId)}/shutdown`, + {}, + ); + return resp.data; + }} + confirmButtonText="Kill task" + successText="Task was killed" + failText="Could not kill task" + intent={Intent.DANGER} + onClose={() => { + this.setState({ killTaskId: undefined }); + }} + onSuccess={() => { + this.taskQueryManager.rerunLastQuery(); + }} + > +

{`Are you sure you want to kill task '${killTaskId}'?`}

+ + ); + } + + private renderTaskFilterableCell(field: string) { + const { taskFilter } = this.state; + + return (row: { value: any }) => ( + this.setState({ taskFilter: filters })} + > + {row.value} + + ); + } + + private onTaskDetail(task: TaskQueryResultRow) { + this.setState({ + taskTableActionDialogId: task.task_id, + taskTableActionDialogStatus: task.status, + taskTableActionDialogActions: this.getTaskActions( + task.task_id, + task.datasource, + task.status, + task.type, + ), + }); + } + + private renderTaskTable() { + const { tasksState, taskFilter, groupTasksBy, hiddenTaskColumns, supervisorFilter } = + this.state; + + const tasks = tasksState.data || []; + return ( + { + this.setState({ + supervisorFilter: + column.id === 'datasource' + ? syncFilterClauseById(supervisorFilter, filtered, 'datasource') + : supervisorFilter, + taskFilter: filtered, + }); + }} + defaultSorted={[{ id: 'status', desc: true }]} + pivotBy={groupTasksBy ? [groupTasksBy] : []} + defaultPageSize={SMALL_TABLE_PAGE_SIZE} + pageSizeOptions={SMALL_TABLE_PAGE_SIZE_OPTIONS} + showPagination={tasks.length > SMALL_TABLE_PAGE_SIZE} + columns={[ + { + Header: 'Task ID', + accessor: 'task_id', + width: 440, + Cell: ({ value, original }) => ( + this.onTaskDetail(original)} + hoverIcon={IconNames.EDIT} + > + {value} + + ), + Aggregated: () => '', + show: hiddenTaskColumns.shown('Task ID'), + }, + { + Header: 'Group ID', + accessor: 'group_id', + width: 300, + Cell: this.renderTaskFilterableCell('group_id'), + Aggregated: () => '', + show: hiddenTaskColumns.shown('Group ID'), + }, + { + Header: 'Type', + accessor: 'type', + width: 140, + Cell: this.renderTaskFilterableCell('type'), + show: hiddenTaskColumns.shown('Type'), + }, + { + Header: 'Datasource', + accessor: 'datasource', + width: 200, + Cell: this.renderTaskFilterableCell('datasource'), + show: hiddenTaskColumns.shown('Datasource'), + }, + { + Header: 'Status', + id: 'status', + width: 110, + accessor: row => ({ + status: row.status, + created_time: row.created_time, + toString: () => row.status, + }), + Cell: row => { + if (row.aggregated) return ''; + const { status } = row.original; + const errorMsg = row.original.error_msg; + return ( + this.setState({ taskFilter: filters })} + > + + ●  + {status} + {errorMsg && errorMsg !== CANCELED_ERROR_MSG && ( + this.setState({ alertErrorMsg: errorMsg })} + title={errorMsg} + > +  ? + + )} + + + ); + }, + sortMethod: (d1, d2) => { + const typeofD1 = typeof d1; + const typeofD2 = typeof d2; + if (typeofD1 !== typeofD2) return 0; + switch (typeofD1) { + case 'string': + return IngestionView.statusRanking[d1] - IngestionView.statusRanking[d2]; + + case 'object': + return ( + IngestionView.statusRanking[d1.status] - + IngestionView.statusRanking[d2.status] || + d1.created_time.localeCompare(d2.created_time) + ); + + default: + return 0; + } + }, + show: hiddenTaskColumns.shown('Status'), + }, + { + Header: 'Created time', + accessor: 'created_time', + width: 190, + Cell: this.renderTaskFilterableCell('created_time'), + Aggregated: () => '', + show: hiddenTaskColumns.shown('Created time'), + }, + { + Header: 'Duration', + accessor: 'duration', + width: 80, + filterable: false, + className: 'padded', + Cell({ value, original, aggregated }) { + if (aggregated) return ''; + if (value > 0) { + return formatDuration(value); + } + if (oneOf(original.status, 'RUNNING', 'PENDING') && original.created_time) { + // Compute running duration from the created time if it exists + return formatDuration(Date.now() - Date.parse(original.created_time)); + } + return ''; + }, + Aggregated: () => '', + show: hiddenTaskColumns.shown('Duration'), + }, + { + Header: 'Location', + accessor: 'location', + width: 200, + Cell: this.renderTaskFilterableCell('location'), + Aggregated: () => '', + show: hiddenTaskColumns.shown('Location'), + }, + { + Header: ACTION_COLUMN_LABEL, + id: ACTION_COLUMN_ID, + accessor: 'task_id', + width: ACTION_COLUMN_WIDTH, + filterable: false, + Cell: row => { + if (row.aggregated) return ''; + const id = row.value; + const type = row.row.type; + const { datasource, status } = row.original; + const taskActions = this.getTaskActions(id, datasource, status, type); + return ( + this.onTaskDetail(row.original)} + actions={taskActions} + /> + ); + }, + Aggregated: () => '', + show: hiddenTaskColumns.shown(ACTION_COLUMN_LABEL), + }, + ]} + /> + ); + } + + renderBulkSupervisorActions() { + const { capabilities, goToQuery } = this.props; + + return ( + <> + + {capabilities.hasSql() && ( + goToQuery({ queryString: IngestionView.SUPERVISOR_SQL })} + /> + )} + this.setState({ supervisorSpecDialogOpen: true })} + /> + this.setState({ showResumeAllSupervisors: true })} + /> + this.setState({ showSuspendAllSupervisors: true })} + /> + this.setState({ showTerminateAllSupervisors: true })} + /> + + {this.renderResumeAllSupervisorAction()} + {this.renderSuspendAllSupervisorAction()} + {this.renderTerminateAllSupervisorAction()} + + ); + } + + renderResumeAllSupervisorAction() { + const { showResumeAllSupervisors } = this.state; + if (!showResumeAllSupervisors) return; + + return ( + { + const resp = await Api.instance.post(`/druid/indexer/v1/supervisor/resumeAll`, {}); + return resp.data; + }} + confirmButtonText="Resume all supervisors" + successText="All supervisors have been resumed" + failText="Could not resume all supervisors" + intent={Intent.PRIMARY} + onClose={() => { + this.setState({ showResumeAllSupervisors: false }); + }} + onSuccess={() => { + this.supervisorQueryManager.rerunLastQuery(); + }} + > +

Are you sure you want to resume all the supervisors?

+ + ); + } + + renderSuspendAllSupervisorAction() { + const { showSuspendAllSupervisors } = this.state; + if (!showSuspendAllSupervisors) return; + + return ( + { + const resp = await Api.instance.post(`/druid/indexer/v1/supervisor/suspendAll`, {}); + return resp.data; + }} + confirmButtonText="Suspend all supervisors" + successText="All supervisors have been suspended" + failText="Could not suspend all supervisors" + intent={Intent.DANGER} + onClose={() => { + this.setState({ showSuspendAllSupervisors: false }); + }} + onSuccess={() => { + this.supervisorQueryManager.rerunLastQuery(); + }} + > +

Are you sure you want to suspend all the supervisors?

+ + ); + } + + renderTerminateAllSupervisorAction() { + const { showTerminateAllSupervisors } = this.state; + if (!showTerminateAllSupervisors) return; + + return ( + { + const resp = await Api.instance.post(`/druid/indexer/v1/supervisor/terminateAll`, {}); + return resp.data; + }} + confirmButtonText="Terminate all supervisors" + successText="All supervisors have been terminated" + failText="Could not terminate all supervisors" + intent={Intent.DANGER} + onClose={() => { + this.setState({ showTerminateAllSupervisors: false }); + }} + onSuccess={() => { + this.supervisorQueryManager.rerunLastQuery(); + }} + > +

Are you sure you want to terminate all the supervisors?

+ + ); + } + + renderBulkTasksActions() { + const { goToQuery, capabilities } = this.props; + + return ( + + {capabilities.hasSql() && ( + goToQuery({ queryString: IngestionView.TASK_SQL })} + /> + )} + this.setState({ taskSpecDialogOpen: true })} + /> + + ); + } + + render(): JSX.Element { + const { + groupTasksBy, + supervisorSpecDialogOpen, + taskSpecDialogOpen, + alertErrorMsg, + taskTableActionDialogId, + taskTableActionDialogActions, + supervisorTableActionDialogId, + supervisorTableActionDialogActions, + taskTableActionDialogStatus, + hiddenSupervisorColumns, + hiddenTaskColumns, + } = this.state; + + return ( + <> + +
+ + { + if (auto && hasPopoverOpen()) return; + this.supervisorQueryManager.rerunLastQuery(auto); + }} + /> + {this.renderBulkSupervisorActions()} + + this.setState(prevState => ({ + hiddenSupervisorColumns: prevState.hiddenSupervisorColumns.toggle(column), + })) + } + tableColumnsHidden={hiddenSupervisorColumns.getHiddenColumns()} + /> + + {this.renderSupervisorTable()} +
+
+ + + + + + + + + + { + if (auto && hasPopoverOpen()) return; + this.taskQueryManager.rerunLastQuery(auto); + }} + /> + {this.renderBulkTasksActions()} + + this.setState(prevState => ({ + hiddenTaskColumns: prevState.hiddenTaskColumns.toggle(column), + })) + } + tableColumnsHidden={hiddenTaskColumns.getHiddenColumns()} + /> + + {this.renderTaskTable()} +
+ + {this.renderResumeSupervisorAction()} + {this.renderSuspendSupervisorAction()} + {this.renderResetSupervisorAction()} + {this.renderTerminateSupervisorAction()} + {this.renderKillTaskAction()} + {supervisorSpecDialogOpen && ( + + )} + {taskSpecDialogOpen && ( + + )} + this.setState({ alertErrorMsg: undefined })} + > +

{alertErrorMsg}

+ + {supervisorTableActionDialogId && ( + this.setState({ supervisorTableActionDialogId: undefined })} + /> + )} + {taskTableActionDialogId && taskTableActionDialogStatus && ( + this.setState({ taskTableActionDialogId: undefined })} + /> + )} + + ); + } +} From 1fa30966de6117fbfd1e577c649fbc947cc96e97 Mon Sep 17 00:00:00 2001 From: Luke Young <91491244+lyoung-confluent@users.noreply.github.com> Date: Thu, 24 Mar 2022 13:52:08 -0700 Subject: [PATCH 051/114] go/codeowners: Generate CODEOWNERS [ci skip] (#87) --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000000..1941fed4a3fd --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# See go/codeowners - automatically generated for confluentinc/druid: +* @confluentinc/observability From ba28327ffa06f4f331142415c39f78b58b54d886 Mon Sep 17 00:00:00 2001 From: Konstantine Karantasis Date: Wed, 31 Aug 2022 23:09:44 -0700 Subject: [PATCH 052/114] fixes in pom.xml files --- extensions-contrib/confluent-extensions/pom.xml | 2 +- extensions-contrib/opencensus-extensions/pom.xml | 2 +- extensions-contrib/opentelemetry-extensions/pom.xml | 2 +- pom.xml | 1 - 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/extensions-contrib/confluent-extensions/pom.xml b/extensions-contrib/confluent-extensions/pom.xml index 001ebc9221bc..3c5256dea45d 100644 --- a/extensions-contrib/confluent-extensions/pom.xml +++ b/extensions-contrib/confluent-extensions/pom.xml @@ -17,7 +17,7 @@ druid org.apache.druid - 0.22.1 + 24.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml index 7f87b2db4c9f..75e1c88803cd 100644 --- a/extensions-contrib/opencensus-extensions/pom.xml +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -31,7 +31,7 @@ druid org.apache.druid - 0.22.1 + 24.0.0-SNAPSHOT ../../pom.xml diff --git a/extensions-contrib/opentelemetry-extensions/pom.xml b/extensions-contrib/opentelemetry-extensions/pom.xml index 63aa274f699d..6d451cacd4c2 100644 --- a/extensions-contrib/opentelemetry-extensions/pom.xml +++ b/extensions-contrib/opentelemetry-extensions/pom.xml @@ -30,7 +30,7 @@ druid org.apache.druid - 0.22.1 + 24.0.0-SNAPSHOT ../../pom.xml diff --git a/pom.xml b/pom.xml index 19113420b5b0..fe5323c4d266 100644 --- a/pom.xml +++ b/pom.xml @@ -230,7 +230,6 @@ extensions-contrib/druid-iceberg-extensions extensions-contrib/opencensus-extensions extensions-contrib/confluent-extensions - extensions-contrib/opentelemetry-emitter extensions-contrib/opentelemetry-extensions distribution From 1872925a618ac4085ae545f62b4da42dd616a4e9 Mon Sep 17 00:00:00 2001 From: Konstantine Karantasis Date: Thu, 1 Sep 2022 08:33:43 -0700 Subject: [PATCH 053/114] adapt to new input argument in ParseException --- .../input/opencensus/protobuf/OpenCensusProtobufReader.java | 2 +- .../protobuf/OpenTelemetryMetricsProtobufReader.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java index 42b110847d9a..1ce97e587b3f 100644 --- a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java @@ -89,7 +89,7 @@ List readAsList() return parseMetric(Metric.parseFrom(source.getBuffer())); } catch (InvalidProtocolBufferException e) { - throw new ParseException(e, "Protobuf message could not be parsed"); + throw new ParseException(null, e, "Protobuf message could not be parsed"); } } diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java index 868e1a529495..20f9cb533891 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java @@ -84,7 +84,7 @@ List readAsList() return parseMetricsData(MetricsData.parseFrom(source.getBuffer())); } catch (InvalidProtocolBufferException e) { - throw new ParseException(e, "Protobuf message could not be parsed"); + throw new ParseException(null, e, "Protobuf message could not be parsed"); } } From 4c9a884db832643bea32c729fe759f89d9c6e6fc Mon Sep 17 00:00:00 2001 From: Konstantine Karantasis Date: Thu, 1 Sep 2022 08:34:15 -0700 Subject: [PATCH 054/114] adapt to the new constructor for DimensionsSpec --- .../io/confluent/druid/transform/ExtractTransformTest.java | 6 +----- .../input/opencensus/protobuf/OpenCensusBenchmark.java | 3 ++- .../protobuf/OpenCensusProtobufInputRowParserTest.java | 4 ++-- .../protobuf/OpenTelemetryMetricsProtobufReaderTest.java | 6 +++--- .../opentelemetry/protobuf/OpenTelemetryBenchmark.java | 3 +-- .../protobuf/OpenTelemetryMetricsProtobufReaderTest.java | 7 +++---- 6 files changed, 12 insertions(+), 17 deletions(-) diff --git a/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java b/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java index 323b48b9c6ad..7fd9fba7d5e9 100644 --- a/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java +++ b/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java @@ -28,11 +28,7 @@ public class ExtractTransformTest private static final MapInputRowParser PARSER = new MapInputRowParser( new TimeAndDimsParseSpec( new TimestampSpec("t", "auto", DateTimes.of("2020-01-01")), - new DimensionsSpec( - DimensionsSpec.getDefaultSchemas(ImmutableList.of("topic", "tenant")), - null, - null - ) + new DimensionsSpec( DimensionsSpec.getDefaultSchemas(ImmutableList.of("topic", "tenant"))) ) ); diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusBenchmark.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusBenchmark.java index 6a0ac4a080b0..871ce0321b8f 100644 --- a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusBenchmark.java +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusBenchmark.java @@ -41,6 +41,7 @@ import java.nio.ByteBuffer; import java.time.Instant; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -54,7 +55,7 @@ public class OpenCensusBenchmark private static final JSONParseSpec PARSE_SPEC = new JSONParseSpec( new TimestampSpec("timestamp", "millis", null), - new DimensionsSpec(null, null, null), + new DimensionsSpec(Collections.emptyList()), new JSONPathSpec( true, Lists.newArrayList( diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java index 2e0266e7dd9b..a9c696cd27cd 100644 --- a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParserTest.java @@ -63,7 +63,7 @@ public class OpenCensusProtobufInputRowParserTest static final JSONParseSpec PARSE_SPEC = new JSONParseSpec( new TimestampSpec("timestamp", "millis", null), - new DimensionsSpec(null, null, null), + new DimensionsSpec(Collections.emptyList()), new JSONPathSpec( true, Lists.newArrayList( @@ -79,7 +79,7 @@ public class OpenCensusProtobufInputRowParserTest new DimensionsSpec(ImmutableList.of( new StringDimensionSchema("foo_key"), new StringDimensionSchema("env_key") - ), null, null), + )), new JSONPathSpec( true, Lists.newArrayList( diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenTelemetryMetricsProtobufReaderTest.java index 60140e2ab31d..0c8902df560b 100644 --- a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenTelemetryMetricsProtobufReaderTest.java +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenTelemetryMetricsProtobufReaderTest.java @@ -81,7 +81,7 @@ public class OpenTelemetryMetricsProtobufReaderTest new StringDimensionSchema("descriptor." + METRIC_ATTRIBUTE_FOO_KEY), new StringDimensionSchema("custom." + RESOURCE_ATTRIBUTE_ENV), new StringDimensionSchema("custom." + RESOURCE_ATTRIBUTE_COUNTRY) - ), null, null); + )); public static final String TOPIC = "telemetry.metrics.otel"; public static final int PARTITION = 2; @@ -291,11 +291,11 @@ public void testDimensionSpecExclusions() throws IOException .setKey(METRIC_ATTRIBUTE_FOO_KEY) .setValue(AnyValue.newBuilder().setStringValue(METRIC_ATTRIBUTE_FOO_VAL).build()).build())); - DimensionsSpec dimensionsSpecWithExclusions = new DimensionsSpec(null, + DimensionsSpec dimensionsSpecWithExclusions = DimensionsSpec.builder().setDimensionExclusions( ImmutableList.of( "descriptor." + METRIC_ATTRIBUTE_COLOR, "custom." + RESOURCE_ATTRIBUTE_COUNTRY - ), null); + )).build(); MetricsData metricsData = metricsDataBuilder.build(); ConsumerRecord consumerRecord = new ConsumerRecord(TOPIC, PARTITION, OFFSET, TS, TSTYPE, diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java index bbb77f1c1a1f..cf5b19b70e4a 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java @@ -72,8 +72,7 @@ public class OpenTelemetryBenchmark new DimensionsSpec(ImmutableList.of( new StringDimensionSchema("name"), new StringDimensionSchema("value"), - new StringDimensionSchema("foo_key")), - null, null), + new StringDimensionSchema("foo_key"))), null); private static final OpenTelemetryMetricsProtobufInputFormat INPUT_FORMAT = diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java index 8a8edafab562..a4d85ece1d79 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java @@ -70,7 +70,7 @@ public class OpenTelemetryMetricsProtobufReaderTest new StringDimensionSchema("descriptor." + METRIC_ATTRIBUTE_FOO_KEY), new StringDimensionSchema("custom." + RESOURCE_ATTRIBUTE_ENV), new StringDimensionSchema("custom." + RESOURCE_ATTRIBUTE_COUNTRY) - ), null, null); + )); @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -256,11 +256,10 @@ public void testDimensionSpecExclusions() MetricsData metricsData = metricsDataBuilder.build(); - DimensionsSpec dimensionsSpecWithExclusions = new DimensionsSpec(null, - ImmutableList.of( + DimensionsSpec dimensionsSpecWithExclusions = DimensionsSpec.builder().setDimensionExclusions(ImmutableList.of( "descriptor." + METRIC_ATTRIBUTE_COLOR, "custom." + RESOURCE_ATTRIBUTE_COUNTRY - ), null); + )).build(); CloseableIterator rows = new OpenTelemetryMetricsProtobufReader( dimensionsSpecWithExclusions, From ed78e9d22420a1162e0375b6ddc38661b85fc7bc Mon Sep 17 00:00:00 2001 From: Yun Fu Date: Fri, 3 Jun 2022 11:12:16 -0700 Subject: [PATCH 055/114] update obs-data team as codeownders (#98) --- .github/CODEOWNERS | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1941fed4a3fd..af377f305262 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1 @@ -# See go/codeowners - automatically generated for confluentinc/druid: -* @confluentinc/observability +* @confluentinc/obs-data From 9ebd2f535560a2d7ab395aa507d52d6e86c00bce Mon Sep 17 00:00:00 2001 From: Naya Chen Date: Mon, 11 Jul 2022 11:19:26 -0700 Subject: [PATCH 056/114] [OBSDATA-334] Patch opencensus/opentelemetry parse exception (#99) --- .../protobuf/OpenCensusProtobufReader.java | 15 +++++++++++- ...java => OpenCensusProtobufReaderTest.java} | 24 ++++++++++++++++++- .../OpenTelemetryMetricsProtobufReader.java | 16 +++++++++++-- ...penTelemetryMetricsProtobufReaderTest.java | 16 +++++++++++++ 4 files changed, 67 insertions(+), 4 deletions(-) rename extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/{OpenTelemetryMetricsProtobufReaderTest.java => OpenCensusProtobufReaderTest.java} (93%) diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java index 1ce97e587b3f..249a5d096d96 100644 --- a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java @@ -19,6 +19,8 @@ package org.apache.druid.data.input.opencensus.protobuf; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -42,6 +44,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -80,7 +83,17 @@ private interface LabelContext @Override public CloseableIterator read() { - return CloseableIterators.withEmptyBaggage(readAsList().iterator()); + Supplier> supplier = Suppliers.memoize(() -> readAsList().iterator()); + return CloseableIterators.withEmptyBaggage(new Iterator() { + @Override + public boolean hasNext() { + return supplier.get().hasNext(); + } + @Override + public InputRow next() { + return supplier.get().next(); + } + }); } List readAsList() diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReaderTest.java similarity index 93% rename from extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenTelemetryMetricsProtobufReaderTest.java rename to extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReaderTest.java index 0c8902df560b..43ef6d455d85 100644 --- a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenTelemetryMetricsProtobufReaderTest.java +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReaderTest.java @@ -32,6 +32,7 @@ import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.data.input.kafka.KafkaRecordEntity; import org.apache.druid.java.util.common.parsers.CloseableIterator; +import org.apache.druid.java.util.common.parsers.ParseException; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.Headers; @@ -52,7 +53,7 @@ import java.util.List; import java.util.concurrent.TimeUnit; -public class OpenTelemetryMetricsProtobufReaderTest +public class OpenCensusProtobufReaderTest { private static final long TIMESTAMP = TimeUnit.MILLISECONDS.toNanos(Instant.parse("2019-07-12T09:30:01.123Z").toEpochMilli()); public static final String RESOURCE_ATTRIBUTE_COUNTRY = "country"; @@ -325,6 +326,27 @@ public void testDimensionSpecExclusions() throws IOException Assert.assertFalse(row.getDimensions().contains("descriptor.color")); } + @Test + public void testInvalidProtobuf() throws IOException { + byte[] invalidProtobuf = new byte[] { 0x00, 0x01 }; + ConsumerRecord consumerRecord = new ConsumerRecord(TOPIC, PARTITION, OFFSET, TS, TSTYPE, + -1L, -1, -1, null, invalidProtobuf, HEADERS); + KafkaRecordEntity kafkaRecordEntity = new KafkaRecordEntity(consumerRecord); + OpenCensusProtobufInputFormat inputFormat = new OpenCensusProtobufInputFormat("metric.name", + null, + "descriptor.", + "custom."); + + CloseableIterator rows = inputFormat.createReader(new InputRowSchema( + new TimestampSpec("timestamp", "iso", null), + dimensionsSpec, + ColumnsFilter.all() + ), kafkaRecordEntity, null).read(); + + Assert.assertThrows(ParseException.class, () -> rows.hasNext()); + Assert.assertThrows(ParseException.class, () -> rows.next()); + } + private void assertDimensionEquals(InputRow row, String dimension, Object expected) { List values = row.getDimension(dimension); diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java index 20f9cb533891..f5d191cbee37 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java @@ -19,6 +19,8 @@ package org.apache.druid.data.input.opentelemetry.protobuf; +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.protobuf.InvalidProtocolBufferException; @@ -40,6 +42,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -47,7 +50,6 @@ public class OpenTelemetryMetricsProtobufReader implements InputEntityReader { - private final ByteEntity source; private final String metricDimension; private final String valueDimension; @@ -75,7 +77,17 @@ public OpenTelemetryMetricsProtobufReader( @Override public CloseableIterator read() { - return CloseableIterators.withEmptyBaggage(readAsList().iterator()); + Supplier> supplier = Suppliers.memoize(() -> readAsList().iterator()); + return CloseableIterators.withEmptyBaggage(new Iterator() { + @Override + public boolean hasNext() { + return supplier.get().hasNext(); + } + @Override + public InputRow next() { + return supplier.get().next(); + } + }); } List readAsList() diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java index a4d85ece1d79..8e36a0b4fc35 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java @@ -30,6 +30,7 @@ import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.java.util.common.parsers.CloseableIterator; +import org.apache.druid.java.util.common.parsers.ParseException; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -342,6 +343,21 @@ public void testUnsupportedValueTypes() assertDimensionEquals(row, "raw.value", "6"); } + @Test + public void testInvalidProtobuf() { + byte[] invalidProtobuf = new byte[] { 0x00, 0x01 }; + CloseableIterator rows = new OpenTelemetryMetricsProtobufReader( + dimensionsSpec, + new ByteEntity(invalidProtobuf), + "metric.name", + "raw.value", + "descriptor.", + "custom." + ).read(); + Assert.assertThrows(ParseException.class, () -> rows.hasNext()); + Assert.assertThrows(ParseException.class, () -> rows.next()); + } + private void assertDimensionEquals(InputRow row, String dimension, Object expected) { List values = row.getDimension(dimension); From e123f2e6fbf172ec85e964131713c161977c21ea Mon Sep 17 00:00:00 2001 From: Yun Fu Date: Thu, 14 Jul 2022 09:23:47 -0700 Subject: [PATCH 057/114] [METRICS-4487] add obs-oncall as codeowners (#101) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index af377f305262..858ee8331184 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @confluentinc/obs-data +* @confluentinc/obs-data @confluentinc/obs-oncall From 89502f6dde56e31f93b6dbb7cb269b44833bce71 Mon Sep 17 00:00:00 2001 From: nlou9 <39046184+nlou9@users.noreply.github.com> Date: Wed, 20 Jul 2022 11:20:49 -0700 Subject: [PATCH 058/114] DP-8085 - Migrate to Sempahore self-hosted agent (#100) --- .semaphore/semaphore.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 260f07e3c98c..d5e68b7c8f58 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -2,8 +2,7 @@ version: v1.0 name: Apache Druid agent: machine: - type: e1-standard-4 - os_image: ubuntu1804 + type: s1-prod-ubuntu20-04-amd64-1 blocks: - name: "Install" @@ -28,6 +27,7 @@ blocks: value: "-DskipTests -Djacoco.skip=true" prologue: commands: + - echo $SEMAPHORE_WORKFLOW_ID - sem-version java 11 - checkout jobs: @@ -44,6 +44,7 @@ blocks: env_vars: *env_vars prologue: commands: + - echo $SEMAPHORE_WORKFLOW_ID - sem-version java 11 - checkout - cache restore From ba07c32befe8a7a43916fcc1058c254046d08e95 Mon Sep 17 00:00:00 2001 From: Naya Chen Date: Wed, 27 Jul 2022 12:46:36 -0700 Subject: [PATCH 059/114] [OBSDATA-334] Patch opentelemetry IllegalStateException for unsupported metric types (#103) --- .../OpenTelemetryMetricsProtobufReader.java | 12 ++++++---- ...penTelemetryMetricsProtobufReaderTest.java | 24 +++++++++++++++++++ 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java index f5d191cbee37..967f278dcb19 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java @@ -35,6 +35,7 @@ import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.java.util.common.CloseableIterators; +import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.common.parsers.CloseableIterator; import org.apache.druid.java.util.common.parsers.ParseException; @@ -50,6 +51,8 @@ public class OpenTelemetryMetricsProtobufReader implements InputEntityReader { + private static final Logger log = new Logger(OpenTelemetryMetricsProtobufReader.class); + private final ByteEntity source; private final String metricDimension; private final String valueDimension; @@ -146,12 +149,11 @@ private List parseMetric(Metric metric, Map resourceAt } // TODO Support HISTOGRAM and SUMMARY metrics case HISTOGRAM: - case SUMMARY: { - inputRows = Collections.emptyList(); - break; - } + case SUMMARY: default: - throw new IllegalStateException("Unexpected value: " + metric.getDataCase()); + log.warn("Metric type " + metric.getDataCase() + " is not supported or deprecated."); + inputRows = Collections.emptyList(); + } return inputRows; } diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java index 8e36a0b4fc35..febb8ab413dd 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java @@ -357,6 +357,30 @@ public void testInvalidProtobuf() { Assert.assertThrows(ParseException.class, () -> rows.hasNext()); Assert.assertThrows(ParseException.class, () -> rows.next()); } + + @Test + public void testInvalidMetricType() { + metricBuilder + .setName("deprecated_intsum") + .getIntSumBuilder() + .addDataPointsBuilder() + .setTimeUnixNano(TIMESTAMP); + + MetricsData metricsData = metricsDataBuilder.build(); + + CloseableIterator rows = new OpenTelemetryMetricsProtobufReader( + dimensionsSpec, + new ByteEntity(metricsData.toByteArray()), + "metric.name", + "raw.value", + "descriptor.", + "custom." + ).read(); + + List rowList = new ArrayList<>(); + rows.forEachRemaining(rowList::add); + Assert.assertEquals(0, rowList.size()); + } private void assertDimensionEquals(InputRow row, String dimension, Object expected) { From a1c82a76f60edcbdd21def521c663657ae0cd53e Mon Sep 17 00:00:00 2001 From: Harini Rajendran Date: Fri, 16 Sep 2022 07:03:19 -0500 Subject: [PATCH 060/114] Fixing checkstyle issues in openncensus and opentelemetry extensions. (#109) --- .../opencensus/protobuf/OpenCensusProtobufReader.java | 6 ++++-- .../opencensus/protobuf/OpenCensusProtobufReaderTest.java | 5 +++-- .../protobuf/OpenTelemetryMetricsProtobufReader.java | 6 ++++-- .../protobuf/OpenTelemetryMetricsProtobufReaderTest.java | 8 +++++--- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java index 249a5d096d96..48a9eae02482 100644 --- a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java @@ -86,11 +86,13 @@ public CloseableIterator read() Supplier> supplier = Suppliers.memoize(() -> readAsList().iterator()); return CloseableIterators.withEmptyBaggage(new Iterator() { @Override - public boolean hasNext() { + public boolean hasNext() + { return supplier.get().hasNext(); } @Override - public InputRow next() { + public InputRow next() + { return supplier.get().next(); } }); diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReaderTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReaderTest.java index 43ef6d455d85..e7a4cf4907ac 100644 --- a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReaderTest.java +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReaderTest.java @@ -327,8 +327,9 @@ public void testDimensionSpecExclusions() throws IOException } @Test - public void testInvalidProtobuf() throws IOException { - byte[] invalidProtobuf = new byte[] { 0x00, 0x01 }; + public void testInvalidProtobuf() throws IOException + { + byte[] invalidProtobuf = new byte[] {0x00, 0x01}; ConsumerRecord consumerRecord = new ConsumerRecord(TOPIC, PARTITION, OFFSET, TS, TSTYPE, -1L, -1, -1, null, invalidProtobuf, HEADERS); KafkaRecordEntity kafkaRecordEntity = new KafkaRecordEntity(consumerRecord); diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java index 967f278dcb19..54635bd4164b 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java @@ -83,11 +83,13 @@ public CloseableIterator read() Supplier> supplier = Suppliers.memoize(() -> readAsList().iterator()); return CloseableIterators.withEmptyBaggage(new Iterator() { @Override - public boolean hasNext() { + public boolean hasNext() + { return supplier.get().hasNext(); } @Override - public InputRow next() { + public InputRow next() + { return supplier.get().next(); } }); diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java index febb8ab413dd..e010d486d885 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java @@ -344,8 +344,9 @@ public void testUnsupportedValueTypes() } @Test - public void testInvalidProtobuf() { - byte[] invalidProtobuf = new byte[] { 0x00, 0x01 }; + public void testInvalidProtobuf() + { + byte[] invalidProtobuf = new byte[] {0x00, 0x01}; CloseableIterator rows = new OpenTelemetryMetricsProtobufReader( dimensionsSpec, new ByteEntity(invalidProtobuf), @@ -359,7 +360,8 @@ public void testInvalidProtobuf() { } @Test - public void testInvalidMetricType() { + public void testInvalidMetricType() + { metricBuilder .setName("deprecated_intsum") .getIntSumBuilder() From f644e7ac7e85b2af7e9bd2ba6bf0e32ff98eded2 Mon Sep 17 00:00:00 2001 From: Konstantine Karantasis Date: Wed, 26 Oct 2022 10:47:41 -0700 Subject: [PATCH 061/114] Remove SNAPSHOT from versions in confluent pom files --- extensions-contrib/confluent-extensions/pom.xml | 2 +- extensions-contrib/opencensus-extensions/pom.xml | 2 +- extensions-contrib/opentelemetry-extensions/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions-contrib/confluent-extensions/pom.xml b/extensions-contrib/confluent-extensions/pom.xml index 3c5256dea45d..71442614f112 100644 --- a/extensions-contrib/confluent-extensions/pom.xml +++ b/extensions-contrib/confluent-extensions/pom.xml @@ -17,7 +17,7 @@ druid org.apache.druid - 24.0.0-SNAPSHOT + 24.0.0 ../../pom.xml diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml index 75e1c88803cd..e7a9250cc06f 100644 --- a/extensions-contrib/opencensus-extensions/pom.xml +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -31,7 +31,7 @@ druid org.apache.druid - 24.0.0-SNAPSHOT + 24.0.0 ../../pom.xml diff --git a/extensions-contrib/opentelemetry-extensions/pom.xml b/extensions-contrib/opentelemetry-extensions/pom.xml index 6d451cacd4c2..495c78e4a45a 100644 --- a/extensions-contrib/opentelemetry-extensions/pom.xml +++ b/extensions-contrib/opentelemetry-extensions/pom.xml @@ -30,7 +30,7 @@ druid org.apache.druid - 24.0.0-SNAPSHOT + 24.0.0 ../../pom.xml From 4cd4223fbb3c5e2e3dbe8de03ae66ba3e1b04b80 Mon Sep 17 00:00:00 2001 From: Harini Rajendran Date: Mon, 31 Oct 2022 17:25:09 -0500 Subject: [PATCH 062/114] Fixing CI/CD in 24.0.0 upgrade branch (#116) --- .semaphore/semaphore.yml | 2 +- .../java/io/confluent/druid/transform/ExtractTransformTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index d5e68b7c8f58..768e57be0b28 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -129,5 +129,5 @@ blocks: - name: "Other Tests" env_vars: - name: MAVEN_PROJECTS - value: '!server,!processing,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-contrib/confluent-extensions' + value: '!server,!processing,!indexing-service,!extensions-core/kafka-indexing-service,!extensions-contrib/confluent-extensions,!integration-tests-ex/cases' commands: *run_tests diff --git a/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java b/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java index 7fd9fba7d5e9..2ca5390e76b9 100644 --- a/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java +++ b/extensions-contrib/confluent-extensions/src/test/java/io/confluent/druid/transform/ExtractTransformTest.java @@ -28,7 +28,7 @@ public class ExtractTransformTest private static final MapInputRowParser PARSER = new MapInputRowParser( new TimeAndDimsParseSpec( new TimestampSpec("t", "auto", DateTimes.of("2020-01-01")), - new DimensionsSpec( DimensionsSpec.getDefaultSchemas(ImmutableList.of("topic", "tenant"))) + new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("topic", "tenant"))) ) ); From 35b47adfac1f9ca18fabf4881e1168ecc78e3c70 Mon Sep 17 00:00:00 2001 From: Harini Rajendran Date: Wed, 2 Nov 2022 11:51:56 -0500 Subject: [PATCH 063/114] OBSDATA-440 Adding SegmentMetadataEvent and publishing them via KafkaSegmentMetadataEmitter (#117) --- .idea/inspectionProfiles/Druid.xml | 508 ------------------ .../extensions-contrib/kafka-emitter.md | 30 +- extensions-contrib/kafka-emitter/pom.xml | 43 +- .../druid/emitter/kafka/KafkaEmitter.java | 69 ++- .../emitter/kafka/KafkaEmitterConfig.java | 46 +- .../src/main/proto/DruidSegmentEvent.proto | 30 ++ .../emitter/kafka/KafkaEmitterConfigTest.java | 13 +- .../druid/emitter/kafka/KafkaEmitterTest.java | 2 +- pom.xml | 5 + .../emitter/service/SegmentMetadataEvent.java | 31 ++ 10 files changed, 232 insertions(+), 545 deletions(-) delete mode 100644 .idea/inspectionProfiles/Druid.xml create mode 100644 extensions-contrib/kafka-emitter/src/main/proto/DruidSegmentEvent.proto diff --git a/.idea/inspectionProfiles/Druid.xml b/.idea/inspectionProfiles/Druid.xml deleted file mode 100644 index d1f72dd2763f..000000000000 --- a/.idea/inspectionProfiles/Druid.xml +++ /dev/null @@ -1,508 +0,0 @@ - - - - \ No newline at end of file diff --git a/docs/development/extensions-contrib/kafka-emitter.md b/docs/development/extensions-contrib/kafka-emitter.md index 40b63ca73afd..637cfcae2bb8 100644 --- a/docs/development/extensions-contrib/kafka-emitter.md +++ b/docs/development/extensions-contrib/kafka-emitter.md @@ -23,7 +23,7 @@ title: "Kafka Emitter" --> -To use this Apache Druid extension, [include](../../configuration/extensions.md#loading-extensions) `kafka-emitter` in the extensions load list. +To use this Apache Druid extension, [include](../../development/extensions.md#loading-extensions) `kafka-emitter` in the extensions load list. ## Introduction @@ -36,26 +36,28 @@ to monitor the status of your Druid cluster with this extension. All the configuration parameters for the Kafka emitter are under `druid.emitter.kafka`. -| Property | Description | Required | Default | -|----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|-----------|-----------------------| -| `druid.emitter.kafka.bootstrap.servers` | Comma-separated Kafka broker. (`[hostname:port],[hostname:port]...`) | yes | none | -| `druid.emitter.kafka.event.types` | Comma-separated event types.
Supported types are `alerts`, `metrics`, `requests`, and `segment_metadata`. | no | `["metrics", "alerts"]` | -| `druid.emitter.kafka.metric.topic` | Kafka topic name for emitter's target to emit service metrics. If `event.types` contains `metrics`, this field cannot be empty. | no | none | -| `druid.emitter.kafka.alert.topic` | Kafka topic name for emitter's target to emit alerts. If `event.types` contains `alerts`, this field cannot empty. | no | none | -| `druid.emitter.kafka.request.topic` | Kafka topic name for emitter's target to emit request logs. If `event.types` contains `requests`, this field cannot be empty. | no | none | -| `druid.emitter.kafka.segmentMetadata.topic` | Kafka topic name for emitter's target to emit segment metadata. If `event.types` contains `segment_metadata`, this field cannot be empty. | no | none | -| `druid.emitter.kafka.producer.config` | JSON configuration to set additional properties to Kafka producer. | no | none | -| `druid.emitter.kafka.clusterName` | Optional value to specify the name of your Druid cluster. It can help make groups in your monitoring environment. | no | none | +| property | description | required? | default | +|----------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------|-----------------------| +| `druid.emitter.kafka.bootstrap.servers` | Comma-separated Kafka broker. (`[hostname:port],[hostname:port]...`) | yes | none | +| `druid.emitter.kafka.event.types` | Comma-separated event types.
Choices: alerts, metrics, requests, segmentMetadata | no | ["metrics", "alerts"] | +| `druid.emitter.kafka.metric.topic` | Kafka topic name for emitter's target to emit service metric. If `event.types` contains `metrics`, this field cannot be left empty | no | none | +| `druid.emitter.kafka.alert.topic` | Kafka topic name for emitter's target to emit alert. If `event.types` contains `alerts`, this field cannot be left empty | no | none | +| `druid.emitter.kafka.request.topic` | Kafka topic name for emitter's target to emit request logs. If `event.types` contains `requests`, this field cannot be left empty | no | none | +| `druid.emitter.kafka.segmentMetadata.topic` | Kafka topic name for emitter's target to emit segments related metadata. If `event.types` contains `segmentMetadata`, this field cannot be left empty | no | none | +| `druid.emitter.kafka.segmentMetadata.topic.format` | Format in which segment related metadata will be emitted.
Choices: json, protobuf.
If set to `protobuf`, then segment metadata is emitted in `DruidSegmentEvent.proto` format | no | json | +| `druid.emitter.kafka.producer.config` | JSON formatted configuration which user want to set additional properties to Kafka producer. | no | none | +| `druid.emitter.kafka.clusterName` | Optional value to specify name of your druid cluster. It can help make groups in your monitoring environment. | no | none | ### Example ``` druid.emitter.kafka.bootstrap.servers=hostname1:9092,hostname2:9092 -druid.emitter.kafka.event.types=["metrics", alerts", "requests", "segment_metadata"] -druid.emitter.kafka.metric.topic=druid-metric +druid.emitter.kafka.event.types=["alerts", "requests", "segmentMetadata"] druid.emitter.kafka.alert.topic=druid-alert druid.emitter.kafka.request.topic=druid-request-logs -druid.emitter.kafka.segmentMetadata.topic=druid-segment-metadata +druid.emitter.kafka.segmentMetadata.topic=druid-segment-metadata +druid.emitter.kafka.segmentMetadata.topic.format=protobuf druid.emitter.kafka.producer.config={"max.block.ms":10000} ``` +Whenever `druid.emitter.kafka.segmentMetadata.topic.format` field is updated, it is recommended to also update `druid.emitter.kafka.segmentMetadata.topic` to avoid the same topic from getting polluted with different formats of segment metadata. diff --git a/extensions-contrib/kafka-emitter/pom.xml b/extensions-contrib/kafka-emitter/pom.xml index 46ca7e6c49e1..6ee40af38f95 100644 --- a/extensions-contrib/kafka-emitter/pom.xml +++ b/extensions-contrib/kafka-emitter/pom.xml @@ -91,7 +91,11 @@ slf4j-api provided - + + joda-time + joda-time + provided + junit junit @@ -116,10 +120,30 @@ test-jar test + + com.google.protobuf + protobuf-java + + + com.google.protobuf + protobuf-java-util + - + + kr.motd.maven + os-maven-plugin + 1.7.0 + + + initialize + + detect + + + + org.owasp dependency-check-maven @@ -127,6 +151,21 @@ true + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.6.1 + + + + compile + + + + + com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} + + diff --git a/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitter.java b/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitter.java index dd8f3665f537..8bd4ecc019f5 100644 --- a/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitter.java +++ b/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitter.java @@ -19,12 +19,13 @@ package org.apache.druid.emitter.kafka; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; +import com.google.protobuf.Timestamp; +import com.google.protobuf.util.Timestamps; import org.apache.druid.emitter.kafka.KafkaEmitterConfig.EventType; import org.apache.druid.emitter.kafka.MemoryBoundLinkedBlockingQueue.ObjectContainer; -import org.apache.druid.java.util.common.StringUtils; +import org.apache.druid.emitter.proto.DruidSegmentEvent; import org.apache.druid.java.util.common.lifecycle.LifecycleStop; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.emitter.core.Emitter; @@ -39,6 +40,7 @@ import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.serialization.ByteArraySerializer; import org.apache.kafka.common.serialization.StringSerializer; import java.util.Properties; @@ -112,7 +114,7 @@ protected Producer setKafkaProducer() Properties props = new Properties(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, config.getBootstrapServers()); props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); - props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); props.put(ProducerConfig.RETRIES_CONFIG, DEFAULT_RETRIES); props.putAll(config.getKafkaProducerConfig()); @@ -140,7 +142,7 @@ public void start() scheduler.schedule(this::sendSegmentMetadataToKafka, sendInterval, TimeUnit.SECONDS); } scheduler.scheduleWithFixedDelay(() -> { - log.info("Message lost counter: metricLost=[%d], alertLost=[%d], requestLost=[%d], segmentMetadataLost=[%d], invalidLost=[%d]", + log.info("Message lost counter: metricLost=[%d], alertLost=[%d], requestLost=[%d], invalidLost=[%d] segmentMetadataLost=[%d]", metricLost.get(), alertLost.get(), requestLost.get(), @@ -193,15 +195,14 @@ public void emit(final Event event) EventMap map = event.toMap(); if (config.getClusterName() != null) { map = map.asBuilder() - .put("clusterName", config.getClusterName()) - .build(); + .put("clusterName", config.getClusterName()) + .build(); } - String resultJson = jsonMapper.writeValueAsString(map); - - ObjectContainer objectContainer = new ObjectContainer<>( - resultJson, - StringUtils.toUtf8(resultJson).length + byte[] resultBytes = jsonMapper.writeValueAsBytes(map); + ObjectContainer objectContainer = new ObjectContainer<>( + resultBytes, + resultBytes.length ); Set eventTypes = config.getEventTypes(); @@ -220,18 +221,62 @@ public void emit(final Event event) } else if (event instanceof SegmentMetadataEvent) { if (!eventTypes.contains(EventType.SEGMENT_METADATA) || !segmentMetadataQueue.offer(objectContainer)) { segmentMetadataLost.incrementAndGet(); + } else { + switch (config.getSegmentMetadataTopicFormat()) { + case PROTOBUF: + resultBytes = convertMetadataEventToProto((SegmentMetadataEvent) event, segmentMetadataLost); + objectContainer = new ObjectContainer<>( + resultBytes, + resultBytes.length + ); + break; + case JSON: + // Do Nothing. We already have the JSON object stored in objectContainer + break; + default: + throw new UnsupportedOperationException("segmentMetadata.topic.format has an invalid value " + config.getSegmentMetadataTopicFormat().toString()); + } + if (!segmentMetadataQueue.offer(objectContainer)) { + segmentMetadataLost.incrementAndGet(); + } } } else { invalidLost.incrementAndGet(); } } - catch (JsonProcessingException e) { + catch (Exception e) { invalidLost.incrementAndGet(); log.warn(e, "Exception while serializing event"); } } } + private byte[] convertMetadataEventToProto(SegmentMetadataEvent event, AtomicLong segmentMetadataLost) + { + try { + Timestamp createdTimeTs = Timestamps.fromMillis(event.getCreatedTime().getMillis()); + Timestamp startTimeTs = Timestamps.fromMillis(event.getStartTime().getMillis()); + Timestamp endTimeTs = Timestamps.fromMillis(event.getEndTime().getMillis()); + + DruidSegmentEvent.Builder druidSegmentEventBuilder = DruidSegmentEvent.newBuilder() + .setDataSource(event.getDataSource()) + .setCreatedTime(createdTimeTs) + .setStartTime(startTimeTs) + .setEndTime(endTimeTs) + .setVersion(event.getVersion()) + .setIsCompacted(event.isCompacted()); + if (config.getClusterName() != null) { + druidSegmentEventBuilder.setClusterName(config.getClusterName()); + } + DruidSegmentEvent druidSegmentEvent = druidSegmentEventBuilder.build(); + return druidSegmentEvent.toByteArray(); + } + catch (Exception e) { + log.warn(e, "Exception while serializing SegmentMetadataEvent"); + throw e; + } + } + @Override public void flush() { diff --git a/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitterConfig.java b/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitterConfig.java index 019edd095ea4..809c7b4d79ca 100644 --- a/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitterConfig.java +++ b/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitterConfig.java @@ -40,7 +40,13 @@ public enum EventType METRICS, ALERTS, REQUESTS, - SEGMENT_METADATA; + SEGMENT_METADATA { + @Override + public String toString() + { + return "segmentMetadata"; + } + }; @JsonValue @Override @@ -57,6 +63,25 @@ public static EventType fromString(String name) } public static final Set DEFAULT_EVENT_TYPES = ImmutableSet.of(EventType.ALERTS, EventType.METRICS); + + public enum SegmentMetadataTopicFormat + { + JSON, + PROTOBUF; + + @JsonValue + @Override + public String toString() + { + return StringUtils.toLowerCase(this.name()); + } + + @JsonCreator + public static SegmentMetadataTopicFormat fromString(String name) + { + return valueOf(StringUtils.toUpperCase(name)); + } + } @JsonProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG) private final String bootstrapServers; @Nullable @JsonProperty("event.types") @@ -69,6 +94,8 @@ public static EventType fromString(String name) private final String requestTopic; @Nullable @JsonProperty("segmentMetadata.topic") private final String segmentMetadataTopic; + @Nullable @JsonProperty("segmentMetadata.topic.format") + private final SegmentMetadataTopicFormat segmentMetadataTopicFormat; @JsonProperty private final String clusterName; @JsonProperty("producer.config") @@ -82,6 +109,7 @@ public KafkaEmitterConfig( @Nullable @JsonProperty("alert.topic") String alertTopic, @Nullable @JsonProperty("request.topic") String requestTopic, @Nullable @JsonProperty("segmentMetadata.topic") String segmentMetadataTopic, + @Nullable @JsonProperty("segmentMetadata.topic.format") SegmentMetadataTopicFormat segmentMetadataTopicFormat, @JsonProperty("clusterName") String clusterName, @JsonProperty("producer.config") @Nullable Map kafkaProducerConfig ) @@ -92,6 +120,7 @@ public KafkaEmitterConfig( this.alertTopic = this.eventTypes.contains(EventType.ALERTS) ? Preconditions.checkNotNull(alertTopic, "druid.emitter.kafka.alert.topic can not be null") : null; this.requestTopic = this.eventTypes.contains(EventType.REQUESTS) ? Preconditions.checkNotNull(requestTopic, "druid.emitter.kafka.request.topic can not be null") : null; this.segmentMetadataTopic = this.eventTypes.contains(EventType.SEGMENT_METADATA) ? Preconditions.checkNotNull(segmentMetadataTopic, "druid.emitter.kafka.segmentMetadata.topic can not be null") : null; + this.segmentMetadataTopicFormat = segmentMetadataTopicFormat == null ? SegmentMetadataTopicFormat.JSON : segmentMetadataTopicFormat; this.clusterName = clusterName; this.kafkaProducerConfig = kafkaProducerConfig == null ? ImmutableMap.of() : kafkaProducerConfig; } @@ -153,6 +182,12 @@ public String getSegmentMetadataTopic() return segmentMetadataTopic; } + @JsonProperty + public SegmentMetadataTopicFormat getSegmentMetadataTopicFormat() + { + return segmentMetadataTopicFormat; + } + @JsonProperty public Map getKafkaProducerConfig() { @@ -183,6 +218,7 @@ public boolean equals(Object o) return false; } + if (getAlertTopic() != null ? !getAlertTopic().equals(that.getAlertTopic()) : that.getAlertTopic() != null) { return false; } @@ -195,6 +231,10 @@ public boolean equals(Object o) return false; } + if (getSegmentMetadataTopicFormat() != null ? !getSegmentMetadataTopicFormat().equals(that.getSegmentMetadataTopicFormat()) : that.getSegmentMetadataTopicFormat() != null) { + return false; + } + if (getClusterName() != null ? !getClusterName().equals(that.getClusterName()) : that.getClusterName() != null) { return false; } @@ -210,6 +250,7 @@ public int hashCode() result = 31 * result + (getAlertTopic() != null ? getAlertTopic().hashCode() : 0); result = 31 * result + (getRequestTopic() != null ? getRequestTopic().hashCode() : 0); result = 31 * result + (getSegmentMetadataTopic() != null ? getSegmentMetadataTopic().hashCode() : 0); + result = 31 * result + (getSegmentMetadataTopicFormat() != null ? getSegmentMetadataTopicFormat().hashCode() : 0); result = 31 * result + (getClusterName() != null ? getClusterName().hashCode() : 0); result = 31 * result + getKafkaProducerConfig().hashCode(); return result; @@ -220,11 +261,12 @@ public String toString() { return "KafkaEmitterConfig{" + "bootstrap.servers='" + bootstrapServers + '\'' + - ", event.types='" + eventTypes + '\'' + + ", event.types='" + eventTypes.toString() + '\'' + ", metric.topic='" + metricTopic + '\'' + ", alert.topic='" + alertTopic + '\'' + ", request.topic='" + requestTopic + '\'' + ", segmentMetadata.topic='" + segmentMetadataTopic + '\'' + + ", segmentMetadata.topic.format='" + segmentMetadataTopicFormat + '\'' + ", clusterName='" + clusterName + '\'' + ", Producer.config=" + kafkaProducerConfig + '}'; diff --git a/extensions-contrib/kafka-emitter/src/main/proto/DruidSegmentEvent.proto b/extensions-contrib/kafka-emitter/src/main/proto/DruidSegmentEvent.proto new file mode 100644 index 000000000000..810ab64f92dd --- /dev/null +++ b/extensions-contrib/kafka-emitter/src/main/proto/DruidSegmentEvent.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; +import "google/protobuf/timestamp.proto"; + +option java_multiple_files = true; +option java_package = "org.apache.druid.emitter.proto"; +option java_outer_classname = "DruidSegmentEventMessage"; + +/* Druid segment Event used by Druid to publish first level segment information. + * The message will be consumed by segment processing app. */ +message DruidSegmentEvent { + string dataSource = 1; + + // When this event was created + google.protobuf.Timestamp createdTime = 2; + + // Start time of the segment + google.protobuf.Timestamp startTime = 3; + + // End time of the segment + google.protobuf.Timestamp endTime = 4; + + // Segment version + string version = 5; + + // Cluster name + string clusterName = 6; + + // Is the segment compacted or not + bool isCompacted = 7; +} diff --git a/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterConfigTest.java b/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterConfigTest.java index c4d5811bcb53..dc02349cb181 100644 --- a/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterConfigTest.java +++ b/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterConfigTest.java @@ -46,7 +46,7 @@ public void setUp() public void testSerDeserKafkaEmitterConfig() throws IOException { KafkaEmitterConfig kafkaEmitterConfig = new KafkaEmitterConfig("hostname", null, "metricTest", - "alertTest", "requestTest", "metadataTest", + "alertTest", "requestTest", "metadataTest", null, "clusterNameTest", ImmutableMap.builder() .put("testKey", "testValue").build() ); @@ -60,7 +60,7 @@ public void testSerDeserKafkaEmitterConfig() throws IOException public void testSerDeserKafkaEmitterConfigNullRequestTopic() throws IOException { KafkaEmitterConfig kafkaEmitterConfig = new KafkaEmitterConfig("hostname", null, "metricTest", - "alertTest", null, "metadataTest", + "alertTest", null, "metadataTest",null, "clusterNameTest", ImmutableMap.builder() .put("testKey", "testValue").build() ); @@ -76,7 +76,7 @@ public void testSerDeserKafkaEmitterConfigNullMetricsTopic() throws IOException Set eventTypeSet = new HashSet(); eventTypeSet.add(KafkaEmitterConfig.EventType.SEGMENT_METADATA); KafkaEmitterConfig kafkaEmitterConfig = new KafkaEmitterConfig("hostname", eventTypeSet, null, - null, null, "metadataTest", + null, null, "metadataTest", null, "clusterNameTest", ImmutableMap.builder() .put("testKey", "testValue").build() ); @@ -90,7 +90,8 @@ public void testSerDeserKafkaEmitterConfigNullMetricsTopic() throws IOException public void testSerDeNotRequiredKafkaProducerConfig() { KafkaEmitterConfig kafkaEmitterConfig = new KafkaEmitterConfig("localhost:9092", null, "metricTest", - "alertTest", null, "metadataTest", + + "alertTest", null, "metadataTest", null, "clusterNameTest", null ); try { @@ -105,9 +106,9 @@ public void testSerDeNotRequiredKafkaProducerConfig() @Test public void testDeserializeEventTypesWithDifferentCase() throws JsonProcessingException { - Assert.assertEquals(KafkaEmitterConfig.EventType.SEGMENT_METADATA, mapper.readValue("\"segment_metadata\"", KafkaEmitterConfig.EventType.class)); + Assert.assertEquals(KafkaEmitterConfig.EventType.SEGMENT_METADATA, mapper.readValue("\"segmentMetadata\"", KafkaEmitterConfig.EventType.class)); Assert.assertEquals(KafkaEmitterConfig.EventType.ALERTS, mapper.readValue("\"alerts\"", KafkaEmitterConfig.EventType.class)); - Assert.assertThrows(ValueInstantiationException.class, () -> mapper.readValue("\"segmentMetadata\"", KafkaEmitterConfig.EventType.class)); + Assert.assertThrows(ValueInstantiationException.class, () -> mapper.readValue("\"segment_metadata\"", KafkaEmitterConfig.EventType.class)); } @Test diff --git a/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterTest.java b/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterTest.java index 9e6846a5d8b8..b5196018d784 100644 --- a/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterTest.java +++ b/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterTest.java @@ -106,7 +106,7 @@ public void testKafkaEmitter() throws InterruptedException ObjectMapper mapper = new ObjectMapper(); mapper.registerModule(new JodaModule()); final KafkaEmitter kafkaEmitter = new KafkaEmitter( - new KafkaEmitterConfig("", eventsType, "metrics", "alerts", requestTopic, "metadata", "test-cluster", null), + new KafkaEmitterConfig("", eventsType, "metrics", "alerts", requestTopic, "metadata", null,"test-cluster", null), mapper ) { diff --git a/pom.xml b/pom.xml index fe5323c4d266..ef203f0ea72e 100644 --- a/pom.xml +++ b/pom.xml @@ -821,6 +821,11 @@ protobuf-java ${protobuf.version} + + com.google.protobuf + protobuf-java-util + ${protobuf.version} + io.tesla.aether tesla-aether diff --git a/processing/src/main/java/org/apache/druid/java/util/emitter/service/SegmentMetadataEvent.java b/processing/src/main/java/org/apache/druid/java/util/emitter/service/SegmentMetadataEvent.java index 7e249f72d0a6..732db1d3cecb 100644 --- a/processing/src/main/java/org/apache/druid/java/util/emitter/service/SegmentMetadataEvent.java +++ b/processing/src/main/java/org/apache/druid/java/util/emitter/service/SegmentMetadataEvent.java @@ -97,6 +97,37 @@ public String getFeed() { return "segment_metadata"; } + + public DateTime getCreatedTime() + { + return createdTime; + } + + public DateTime getStartTime() + { + return startTime; + } + + public DateTime getEndTime() + { + return endTime; + } + + public String getDataSource() + { + return dataSource; + } + + public String getVersion() + { + return version; + } + + public boolean isCompacted() + { + return isCompacted; + } + @Override @JsonValue public EventMap toMap() From 0268082609ca8ef658a785c0ea205dea55381765 Mon Sep 17 00:00:00 2001 From: Naya Chen Date: Wed, 2 Nov 2022 18:01:08 +0000 Subject: [PATCH 064/114] Change unsupported type message from WARN to TRACE (#119) --- .../protobuf/OpenTelemetryMetricsProtobufReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java index 54635bd4164b..3201b1c26179 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java @@ -153,7 +153,7 @@ private List parseMetric(Metric metric, Map resourceAt case HISTOGRAM: case SUMMARY: default: - log.warn("Metric type " + metric.getDataCase() + " is not supported or deprecated."); + log.trace("Metric type " + metric.getDataCase() + " is not supported or deprecated."); inputRows = Collections.emptyList(); } From df617fc299e2806b3765cf2e25edc7efce70e516 Mon Sep 17 00:00:00 2001 From: Naya Chen Date: Wed, 2 Nov 2022 20:41:26 +0000 Subject: [PATCH 065/114] Use place holder for logging invalid format (#120) Use place holder for logging invalid format for better performance --- .../protobuf/OpenTelemetryMetricsProtobufReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java index 3201b1c26179..27d424848ea8 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java @@ -153,7 +153,7 @@ private List parseMetric(Metric metric, Map resourceAt case HISTOGRAM: case SUMMARY: default: - log.trace("Metric type " + metric.getDataCase() + " is not supported or deprecated."); + log.trace("Metric type {} is not supported", metric.getDataCase()); inputRows = Collections.emptyList(); } From f6cf31d3bd4415fb1bbbae2b2cd8af3ad2f2bc41 Mon Sep 17 00:00:00 2001 From: Corey Christous Date: Wed, 2 Nov 2022 20:16:03 -0400 Subject: [PATCH 066/114] DP-9370 - use cc-service-bot to manage Semaphore project (#118) --- service.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 service.yml diff --git a/service.yml b/service.yml new file mode 100644 index 000000000000..71638433d4a4 --- /dev/null +++ b/service.yml @@ -0,0 +1,11 @@ +name: druid +lang: unknown +lang_version: unknown +git: + enable: true +github: + enable: true +semaphore: + enable: true + pipeline_enable: false + branches: [] From b7a178cc2755bf9e7e086789478c50046adaa78c Mon Sep 17 00:00:00 2001 From: Confluent Jenkins Bot Date: Wed, 23 Nov 2022 06:20:31 +0000 Subject: [PATCH 067/114] chore: update repo semaphore project --- .semaphore/project.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .semaphore/project.yml diff --git a/.semaphore/project.yml b/.semaphore/project.yml new file mode 100644 index 000000000000..2987c6244b20 --- /dev/null +++ b/.semaphore/project.yml @@ -0,0 +1,33 @@ +apiVersion: v1alpha +kind: Project +metadata: + name: druid + description: "" +spec: + visibility: private + repository: + url: git@github.com:confluentinc/druid.git + run_on: + - branches + - tags + - pull_requests + pipeline_file: .semaphore/semaphore.yml + integration_type: github_app + status: + pipeline_files: + - path: .semaphore/semaphore.yml + level: pipeline + custom_permissions: true + debug_permissions: + - empty + - default_branch + - non_default_branch + - pull_request + - forked_pull_request + - tag + attach_permissions: + - default_branch + - non_default_branch + - pull_request + - forked_pull_request + - tag From 10fc98a6caf9dd055befa4fc452469154db28350 Mon Sep 17 00:00:00 2001 From: Corey Christous Date: Wed, 30 Nov 2022 14:52:00 -0500 Subject: [PATCH 068/114] DP-9632: remediate duplicate Semaphore workflows (#121) Only build the master branch and the `x.x.x-confluent` Druid release branches by default --- service.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/service.yml b/service.yml index 71638433d4a4..6ac069c67d9b 100644 --- a/service.yml +++ b/service.yml @@ -8,4 +8,6 @@ github: semaphore: enable: true pipeline_enable: false - branches: [] + branches: + - master + - /^.*-confluent$/ From ba98b822d3d1697159f7b4ad5e12458cf1f66878 Mon Sep 17 00:00:00 2001 From: Confluent Jenkins Bot Date: Thu, 1 Dec 2022 06:16:20 +0000 Subject: [PATCH 069/114] chore: update repo semaphore project --- .semaphore/project.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.semaphore/project.yml b/.semaphore/project.yml index 2987c6244b20..b5669343e76c 100644 --- a/.semaphore/project.yml +++ b/.semaphore/project.yml @@ -17,6 +17,10 @@ spec: pipeline_files: - path: .semaphore/semaphore.yml level: pipeline + whitelist: + branches: + - master + - /^.*-confluent$/ custom_permissions: true debug_permissions: - empty From a7362880507b4094a978eae673d85a80b5bb72fe Mon Sep 17 00:00:00 2001 From: Konstantine Karantasis Date: Tue, 13 Dec 2022 09:53:00 -0800 Subject: [PATCH 070/114] Bump version to 24.0.1 in confluent extensions after rebasing on top of druid-24.0.1 --- extensions-contrib/confluent-extensions/pom.xml | 2 +- extensions-contrib/opencensus-extensions/pom.xml | 2 +- extensions-contrib/opentelemetry-extensions/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions-contrib/confluent-extensions/pom.xml b/extensions-contrib/confluent-extensions/pom.xml index 71442614f112..45d9c4c96a2b 100644 --- a/extensions-contrib/confluent-extensions/pom.xml +++ b/extensions-contrib/confluent-extensions/pom.xml @@ -17,7 +17,7 @@ druid org.apache.druid - 24.0.0 + 24.0.1 ../../pom.xml diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml index e7a9250cc06f..081597a0971c 100644 --- a/extensions-contrib/opencensus-extensions/pom.xml +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -31,7 +31,7 @@ druid org.apache.druid - 24.0.0 + 24.0.1 ../../pom.xml diff --git a/extensions-contrib/opentelemetry-extensions/pom.xml b/extensions-contrib/opentelemetry-extensions/pom.xml index 495c78e4a45a..bc63b2189138 100644 --- a/extensions-contrib/opentelemetry-extensions/pom.xml +++ b/extensions-contrib/opentelemetry-extensions/pom.xml @@ -30,7 +30,7 @@ druid org.apache.druid - 24.0.0 + 24.0.1 ../../pom.xml From a2cbcce712891234b8de4b5e0f352ccc0d5fd68d Mon Sep 17 00:00:00 2001 From: Konstantine Karantasis Date: Tue, 3 Jan 2023 12:24:16 -0800 Subject: [PATCH 071/114] Bump version to 24.0.2 in confluent extensions after rebasing on top of druid-24.0.2 --- extensions-contrib/confluent-extensions/pom.xml | 2 +- extensions-contrib/opencensus-extensions/pom.xml | 2 +- extensions-contrib/opentelemetry-extensions/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions-contrib/confluent-extensions/pom.xml b/extensions-contrib/confluent-extensions/pom.xml index 45d9c4c96a2b..e51a72cd986b 100644 --- a/extensions-contrib/confluent-extensions/pom.xml +++ b/extensions-contrib/confluent-extensions/pom.xml @@ -17,7 +17,7 @@ druid org.apache.druid - 24.0.1 + 24.0.2 ../../pom.xml diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml index 081597a0971c..3962bb3131a1 100644 --- a/extensions-contrib/opencensus-extensions/pom.xml +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -31,7 +31,7 @@ druid org.apache.druid - 24.0.1 + 24.0.2 ../../pom.xml diff --git a/extensions-contrib/opentelemetry-extensions/pom.xml b/extensions-contrib/opentelemetry-extensions/pom.xml index bc63b2189138..76bd2d9c1846 100644 --- a/extensions-contrib/opentelemetry-extensions/pom.xml +++ b/extensions-contrib/opentelemetry-extensions/pom.xml @@ -30,7 +30,7 @@ druid org.apache.druid - 24.0.1 + 24.0.2 ../../pom.xml From 9ac69f95ad9726a8eeccaa8dd6bfcdb8cd898460 Mon Sep 17 00:00:00 2001 From: Konstantine Karantasis Date: Wed, 4 Jan 2023 17:42:13 -0800 Subject: [PATCH 072/114] OBSDATA-483: Adapt OpenCensus and OpenTelemetry extensions to the introduction of SettableByteEntity (#113) * OBSDATA-483: Adapt opencensus extension to the introduction of SettableByteEntity * OBSDATA-483: Adapt opentelemetry extension to the introduction of SettableByteEntity * OBSDATA-483: Decide which reader to instantiate on read between opencensus and opentelemetry * OBSDATA-483: Add logger config in opencensus tests * OBSDATA-483: Fix issue with opening the byte entity * OBSDATA-483: Instantiate the right iterator in every read request * OBSDATA-483: Add comments * OBSDATA-483: Address Xavier's comments * OBSDATA-483: Remove unused member fields * OBSDATA-483: Rename enum * OBSDATA-483: Fix trace log to actually print the argument * OBSDATA-483: Keep passing the underlying byte buffer and move its position explicitly * OBSDATA-483: Fix checkstyle issues * OBSDATA-483: Add back handling of InvalidProtocolBufferException * OBSDATA-483: Extend the semaphore workflow execution time to 2 hours * Revert "OBSDATA-483: Extend the semaphore workflow execution time to 2 hours" * OBSDATA-483: Don't close iterator in sample --- .../opencensus-extensions/pom.xml | 6 + .../protobuf/HybridProtobufReader.java | 137 ++++++++++++ .../OpenCensusProtobufInputFormat.java | 54 ++--- .../OpenCensusProtobufInputRowParser.java | 5 +- .../protobuf/OpenCensusProtobufReader.java | 13 +- .../OpenCensusProtobufReaderTest.java | 202 +++++++++--------- .../src/test/resources/log4j2.xml | 35 +++ .../opentelemetry-extensions/pom.xml | 6 + ...enTelemetryMetricsProtobufInputFormat.java | 13 +- .../OpenTelemetryMetricsProtobufReader.java | 15 +- ...penTelemetryMetricsProtobufReaderTest.java | 44 ++-- 11 files changed, 372 insertions(+), 158 deletions(-) create mode 100644 extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/HybridProtobufReader.java create mode 100644 extensions-contrib/opencensus-extensions/src/test/resources/log4j2.xml diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml index 3962bb3131a1..2d0d773c7962 100644 --- a/extensions-contrib/opencensus-extensions/pom.xml +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -55,6 +55,12 @@ ${project.parent.version} provided + + org.apache.druid + druid-indexing-service + ${project.parent.version} + provided + io.opentelemetry.proto opentelemetry-proto diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/HybridProtobufReader.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/HybridProtobufReader.java new file mode 100644 index 000000000000..83c5c20299aa --- /dev/null +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/HybridProtobufReader.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.data.input.opencensus.protobuf; + +import org.apache.druid.data.input.InputEntityReader; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.data.input.InputRowListPlusRawValues; +import org.apache.druid.data.input.KafkaUtils; +import org.apache.druid.data.input.MapBasedInputRow; +import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryMetricsProtobufReader; +import org.apache.druid.indexing.seekablestream.SettableByteEntity; +import org.apache.druid.java.util.common.parsers.CloseableIterator; + +import java.io.IOException; +import java.lang.invoke.MethodHandle; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class HybridProtobufReader implements InputEntityReader +{ + private static final String VERSION_HEADER_KEY = "v"; + private static final int OPENTELEMETRY_FORMAT_VERSION = 1; + + private final DimensionsSpec dimensionsSpec; + private final SettableByteEntity source; + private final String metricDimension; + private final String valueDimension; + private final String metricLabelPrefix; + private final String resourceLabelPrefix; + + private volatile MethodHandle getHeaderMethod = null; + + enum ProtobufReader + { + OPENCENSUS, + OPENTELEMETRY + } + + public HybridProtobufReader( + DimensionsSpec dimensionsSpec, + SettableByteEntity source, + String metricDimension, + String valueDimension, + String metricLabelPrefix, + String resourceLabelPrefix + ) + { + this.dimensionsSpec = dimensionsSpec; + this.source = source; + this.metricDimension = metricDimension; + this.valueDimension = valueDimension; + this.metricLabelPrefix = metricLabelPrefix; + this.resourceLabelPrefix = resourceLabelPrefix; + } + + @Override + public CloseableIterator read() throws IOException + { + return newReader(whichReader()).read(); + } + + public InputEntityReader newReader(ProtobufReader which) + { + switch (which) { + case OPENTELEMETRY: + return new OpenTelemetryMetricsProtobufReader( + dimensionsSpec, + source, + metricDimension, + valueDimension, + metricLabelPrefix, + resourceLabelPrefix + ); + case OPENCENSUS: + default: + return new OpenCensusProtobufReader( + dimensionsSpec, + source, + metricDimension, + metricLabelPrefix, + resourceLabelPrefix + ); + } + } + + public ProtobufReader whichReader() + { + // assume InputEntity is always defined in a single classloader (the kafka-indexing-service classloader) + // so we only have to look it up once. To be completely correct we should cache the method based on classloader + if (getHeaderMethod == null) { + getHeaderMethod = KafkaUtils.lookupGetHeaderMethod( + source.getEntity().getClass().getClassLoader(), + VERSION_HEADER_KEY + ); + } + + try { + byte[] versionHeader = (byte[]) getHeaderMethod.invoke(source.getEntity()); + if (versionHeader != null) { + int version = + ByteBuffer.wrap(versionHeader).order(ByteOrder.LITTLE_ENDIAN).getInt(); + if (version == OPENTELEMETRY_FORMAT_VERSION) { + return ProtobufReader.OPENTELEMETRY; + } + } + } + catch (Throwable t) { + // assume input is opencensus if something went wrong + } + return ProtobufReader.OPENCENSUS; + } + + @Override + public CloseableIterator sample() throws IOException + { + return read().map(row -> InputRowListPlusRawValues.of(row, ((MapBasedInputRow) row).getEvent())); + } +} diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputFormat.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputFormat.java index ae919293cc1e..f06d6bb9deb5 100644 --- a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputFormat.java +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputFormat.java @@ -24,16 +24,12 @@ import org.apache.druid.data.input.InputEntityReader; import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.InputRowSchema; -import org.apache.druid.data.input.KafkaUtils; import org.apache.druid.data.input.impl.ByteEntity; -import org.apache.druid.data.input.opentelemetry.protobuf.OpenTelemetryMetricsProtobufReader; +import org.apache.druid.indexing.seekablestream.SettableByteEntity; import org.apache.druid.java.util.common.StringUtils; import javax.annotation.Nullable; import java.io.File; -import java.lang.invoke.MethodHandle; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.util.Objects; public class OpenCensusProtobufInputFormat implements InputFormat @@ -41,16 +37,12 @@ public class OpenCensusProtobufInputFormat implements InputFormat private static final String DEFAULT_METRIC_DIMENSION = "name"; private static final String DEFAULT_RESOURCE_PREFIX = "resource."; private static final String DEFAULT_VALUE_DIMENSION = "value"; - private static final String VERSION_HEADER_KEY = "v"; - private static final int OPENTELEMETRY_FORMAT_VERSION = 1; private final String metricDimension; private final String valueDimension; private final String metricLabelPrefix; private final String resourceLabelPrefix; - private volatile MethodHandle getHeaderMethod = null; - public OpenCensusProtobufInputFormat( @JsonProperty("metricDimension") String metricDimension, @JsonProperty("valueDimension") @Nullable String valueDimension, @@ -73,41 +65,21 @@ public boolean isSplittable() @Override public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory) { - // assume InputEntity is always defined in a single classloader (the kafka-indexing-service classloader) - // so we only have to look it up once. To be completely correct we should cache the method based on classloader - if (getHeaderMethod == null) { - getHeaderMethod = KafkaUtils.lookupGetHeaderMethod( - source.getClass().getClassLoader(), - OpenCensusProtobufInputFormat.VERSION_HEADER_KEY - ); - } - - try { - byte[] versionHeader = (byte[]) getHeaderMethod.invoke(source); - if (versionHeader != null) { - int version = - ByteBuffer.wrap(versionHeader).order(ByteOrder.LITTLE_ENDIAN).getInt(); - if (version == OPENTELEMETRY_FORMAT_VERSION) { - return new OpenTelemetryMetricsProtobufReader( - inputRowSchema.getDimensionsSpec(), - (ByteEntity) source, - metricDimension, - valueDimension, - metricLabelPrefix, - resourceLabelPrefix - ); - } - } + // Sampler passes a KafkaRecordEntity directly, while the normal code path wraps the same entity in a + // SettableByteEntity + SettableByteEntity settableEntity; + if (source instanceof SettableByteEntity) { + settableEntity = (SettableByteEntity) source; + } else { + SettableByteEntity wrapper = new SettableByteEntity<>(); + wrapper.setEntity((ByteEntity) source); + settableEntity = wrapper; } - catch (Throwable t) { - // assume input is opencensus if something went wrong - } - - - return new OpenCensusProtobufReader( + return new HybridProtobufReader( inputRowSchema.getDimensionsSpec(), - (ByteEntity) source, + settableEntity, metricDimension, + valueDimension, metricLabelPrefix, resourceLabelPrefix ); diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java index 227a6ea5fb8b..e39ca60764b6 100644 --- a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufInputRowParser.java @@ -26,6 +26,7 @@ import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.ParseSpec; +import org.apache.druid.indexing.seekablestream.SettableByteEntity; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.logger.Logger; @@ -103,9 +104,11 @@ public OpenCensusProtobufInputRowParser withParseSpec(ParseSpec parseSpec) @Override public List parseBatch(ByteBuffer input) { + SettableByteEntity settableByteEntity = new SettableByteEntity<>(); + settableByteEntity.setEntity(new ByteEntity(input)); return new OpenCensusProtobufReader( parseSpec.getDimensionsSpec(), - new ByteEntity(input), + settableByteEntity, metricDimension, metricLabelPrefix, resourceLabelPrefix diff --git a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java index 48a9eae02482..c19005e6cb8f 100644 --- a/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java +++ b/extensions-contrib/opencensus-extensions/src/main/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReader.java @@ -36,11 +36,13 @@ import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.indexing.seekablestream.SettableByteEntity; import org.apache.druid.java.util.common.CloseableIterators; import org.apache.druid.java.util.common.parsers.CloseableIterator; import org.apache.druid.java.util.common.parsers.ParseException; import org.apache.druid.utils.CollectionUtils; +import java.nio.ByteBuffer; import java.time.Instant; import java.util.ArrayList; import java.util.HashSet; @@ -55,14 +57,14 @@ public class OpenCensusProtobufReader implements InputEntityReader private static final String VALUE_COLUMN = "value"; private final DimensionsSpec dimensionsSpec; - private final ByteEntity source; + private final SettableByteEntity source; private final String metricDimension; private final String metricLabelPrefix; private final String resourceLabelPrefix; public OpenCensusProtobufReader( DimensionsSpec dimensionsSpec, - ByteEntity source, + SettableByteEntity source, String metricDimension, String metricLabelPrefix, String resourceLabelPrefix @@ -101,7 +103,12 @@ public InputRow next() List readAsList() { try { - return parseMetric(Metric.parseFrom(source.getBuffer())); + ByteBuffer buffer = source.getEntity().getBuffer(); + List rows = parseMetric(Metric.parseFrom(buffer)); + // Explicitly move the position assuming that all the remaining bytes have been consumed because the protobuf + // parser does not update the position itself + buffer.position(buffer.limit()); + return rows; } catch (InvalidProtocolBufferException e) { throw new ParseException(null, e, "Protobuf message could not be parsed"); diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReaderTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReaderTest.java index e7a4cf4907ac..d79f3fdb9f41 100644 --- a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReaderTest.java +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReaderTest.java @@ -25,12 +25,14 @@ import io.opentelemetry.proto.metrics.v1.Metric; import io.opentelemetry.proto.metrics.v1.MetricsData; import org.apache.druid.data.input.ColumnsFilter; +import org.apache.druid.data.input.InputEntityReader; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRowSchema; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.StringDimensionSchema; import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.data.input.kafka.KafkaRecordEntity; +import org.apache.druid.indexing.seekablestream.SettableByteEntity; import org.apache.druid.java.util.common.parsers.CloseableIterator; import org.apache.druid.java.util.common.parsers.ParseException; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -41,9 +43,7 @@ import org.apache.kafka.common.record.TimestampType; import org.junit.Assert; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import java.io.IOException; import java.nio.ByteBuffer; @@ -51,6 +51,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.concurrent.TimeUnit; public class OpenCensusProtobufReaderTest @@ -96,11 +97,6 @@ public class OpenCensusProtobufReaderTest private static final Header HEADERV1 = new RecordHeader("v", V0_HEADER_BYTES); private static final Headers HEADERS = new RecordHeaders(new Header[]{HEADERV1}); - - - @Rule - public ExpectedException expectedException = ExpectedException.none(); - @Before public void setUp() { @@ -124,40 +120,46 @@ public void setUp() public void testSumWithAttributes() throws IOException { metricBuilder - .setName("example_sum") - .getSumBuilder() - .addDataPointsBuilder() - .setAsInt(6) - .setTimeUnixNano(TIMESTAMP) - .addAttributesBuilder() // test sum with attributes - .setKey(METRIC_ATTRIBUTE_COLOR) - .setValue(AnyValue.newBuilder().setStringValue(METRIC_ATTRIBUTE_VALUE_RED).build()); + .setName("example_sum") + .getSumBuilder() + .addDataPointsBuilder() + .setAsInt(6) + .setTimeUnixNano(TIMESTAMP) + .addAttributesBuilder() // test sum with attributes + .setKey(METRIC_ATTRIBUTE_COLOR) + .setValue(AnyValue.newBuilder().setStringValue(METRIC_ATTRIBUTE_VALUE_RED).build()); + MetricsData metricsData = metricsDataBuilder.build(); - ConsumerRecord consumerRecord = new ConsumerRecord(TOPIC, PARTITION, OFFSET, TS, TSTYPE, - -1L, -1, -1, null, metricsData.toByteArray(), HEADERS); - KafkaRecordEntity kafkaRecordEntity = new KafkaRecordEntity(consumerRecord); - OpenCensusProtobufInputFormat inputFormat = new OpenCensusProtobufInputFormat("metric.name", + ConsumerRecord consumerRecord = new ConsumerRecord<>(TOPIC, PARTITION, OFFSET, TS, TSTYPE, -1, -1, + null, metricsData.toByteArray(), HEADERS, Optional.empty()); + OpenCensusProtobufInputFormat inputFormat = new OpenCensusProtobufInputFormat( + "metric.name", null, "descriptor.", - "custom."); + "custom." + ); - CloseableIterator rows = inputFormat.createReader(new InputRowSchema( + SettableByteEntity entity = new SettableByteEntity<>(); + InputEntityReader reader = inputFormat.createReader(new InputRowSchema( new TimestampSpec("timestamp", "iso", null), dimensionsSpec, ColumnsFilter.all() - ), kafkaRecordEntity, null).read(); - - List rowList = new ArrayList<>(); - rows.forEachRemaining(rowList::add); - Assert.assertEquals(1, rowList.size()); - - InputRow row = rowList.get(0); - Assert.assertEquals(4, row.getDimensions().size()); - assertDimensionEquals(row, "metric.name", "example_sum"); - assertDimensionEquals(row, "custom.country", "usa"); - assertDimensionEquals(row, "descriptor.color", "red"); - assertDimensionEquals(row, "value", "6"); + ), entity, null); + + entity.setEntity(new KafkaRecordEntity(consumerRecord)); + try (CloseableIterator rows = reader.read()) { + List rowList = new ArrayList<>(); + rows.forEachRemaining(rowList::add); + Assert.assertEquals(1, rowList.size()); + + InputRow row = rowList.get(0); + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "metric.name", "example_sum"); + assertDimensionEquals(row, "custom.country", "usa"); + assertDimensionEquals(row, "descriptor.color", "red"); + assertDimensionEquals(row, "value", "6"); + } } @Test @@ -173,27 +175,30 @@ public void testGaugeWithAttributes() throws IOException .setValue(AnyValue.newBuilder().setStringValue(METRIC_ATTRIBUTE_VALUE_RED).build()); MetricsData metricsData = metricsDataBuilder.build(); - ConsumerRecord consumerRecord = new ConsumerRecord(TOPIC, PARTITION, OFFSET, TS, TSTYPE, - -1L, -1, -1, null, metricsData.toByteArray(), HEADERS); - KafkaRecordEntity kafkaRecordEntity = new KafkaRecordEntity(consumerRecord); + ConsumerRecord consumerRecord = new ConsumerRecord<>(TOPIC, PARTITION, OFFSET, TS, TSTYPE, -1, -1, + null, metricsData.toByteArray(), HEADERS, Optional.empty()); OpenCensusProtobufInputFormat inputFormat = new OpenCensusProtobufInputFormat("metric.name", null, "descriptor.", "custom."); - CloseableIterator rows = inputFormat.createReader(new InputRowSchema( + SettableByteEntity entity = new SettableByteEntity<>(); + InputEntityReader reader = inputFormat.createReader(new InputRowSchema( new TimestampSpec("timestamp", "iso", null), dimensionsSpec, ColumnsFilter.all() - ), kafkaRecordEntity, null).read(); - - Assert.assertTrue(rows.hasNext()); - InputRow row = rows.next(); - - Assert.assertEquals(4, row.getDimensions().size()); - assertDimensionEquals(row, "metric.name", "example_gauge"); - assertDimensionEquals(row, "custom.country", "usa"); - assertDimensionEquals(row, "descriptor.color", "red"); - assertDimensionEquals(row, "value", "6"); + ), entity, null); + + entity.setEntity(new KafkaRecordEntity(consumerRecord)); + try (CloseableIterator rows = reader.read()) { + Assert.assertTrue(rows.hasNext()); + InputRow row = rows.next(); + + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "metric.name", "example_gauge"); + assertDimensionEquals(row, "custom.country", "usa"); + assertDimensionEquals(row, "descriptor.color", "red"); + assertDimensionEquals(row, "value", "6"); + } } @Test @@ -236,38 +241,38 @@ public void testBatchedMetricParse() throws IOException .setValue(AnyValue.newBuilder().setStringValue(METRIC_ATTRIBUTE_FOO_VAL).build()); MetricsData metricsData = metricsDataBuilder.build(); - ConsumerRecord consumerRecord = new ConsumerRecord(TOPIC, PARTITION, OFFSET, TS, TSTYPE, - -1L, -1, -1, null, metricsData.toByteArray(), HEADERS); - KafkaRecordEntity kafkaRecordEntity = new KafkaRecordEntity(consumerRecord); + ConsumerRecord consumerRecord = new ConsumerRecord<>(TOPIC, PARTITION, OFFSET, TS, TSTYPE, -1, -1, + null, metricsData.toByteArray(), HEADERS, Optional.empty()); OpenCensusProtobufInputFormat inputFormat = new OpenCensusProtobufInputFormat("metric.name", null, "descriptor.", "custom."); - - CloseableIterator rows = inputFormat.createReader(new InputRowSchema( + SettableByteEntity entity = new SettableByteEntity<>(); + InputEntityReader reader = inputFormat.createReader(new InputRowSchema( new TimestampSpec("timestamp", "iso", null), dimensionsSpec, ColumnsFilter.all() - ), kafkaRecordEntity, null).read(); - - - Assert.assertTrue(rows.hasNext()); - InputRow row = rows.next(); - - Assert.assertEquals(4, row.getDimensions().size()); - assertDimensionEquals(row, "metric.name", "example_sum"); - assertDimensionEquals(row, "custom.country", "usa"); - assertDimensionEquals(row, "descriptor.color", "red"); - assertDimensionEquals(row, "value", "6"); - - Assert.assertTrue(rows.hasNext()); - row = rows.next(); - Assert.assertEquals(4, row.getDimensions().size()); - assertDimensionEquals(row, "metric.name", "example_gauge"); - assertDimensionEquals(row, "custom.env", "devel"); - assertDimensionEquals(row, "descriptor.foo_key", "foo_value"); - assertDimensionEquals(row, "value", "8"); - + ), entity, null); + + entity.setEntity(new KafkaRecordEntity(consumerRecord)); + try (CloseableIterator rows = reader.read()) { + Assert.assertTrue(rows.hasNext()); + InputRow row = rows.next(); + + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "metric.name", "example_sum"); + assertDimensionEquals(row, "custom.country", "usa"); + assertDimensionEquals(row, "descriptor.color", "red"); + assertDimensionEquals(row, "value", "6"); + + Assert.assertTrue(rows.hasNext()); + row = rows.next(); + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "metric.name", "example_gauge"); + assertDimensionEquals(row, "custom.env", "devel"); + assertDimensionEquals(row, "descriptor.foo_key", "foo_value"); + assertDimensionEquals(row, "value", "8"); + } } @Test @@ -299,53 +304,58 @@ public void testDimensionSpecExclusions() throws IOException )).build(); MetricsData metricsData = metricsDataBuilder.build(); - ConsumerRecord consumerRecord = new ConsumerRecord(TOPIC, PARTITION, OFFSET, TS, TSTYPE, - -1L, -1, -1, null, metricsData.toByteArray(), HEADERS); - KafkaRecordEntity kafkaRecordEntity = new KafkaRecordEntity(consumerRecord); + ConsumerRecord consumerRecord = new ConsumerRecord<>(TOPIC, PARTITION, OFFSET, TS, TSTYPE, -1, -1, + null, metricsData.toByteArray(), HEADERS, Optional.empty()); OpenCensusProtobufInputFormat inputFormat = new OpenCensusProtobufInputFormat("metric.name", null, "descriptor.", "custom."); - CloseableIterator rows = inputFormat.createReader(new InputRowSchema( + SettableByteEntity entity = new SettableByteEntity<>(); + InputEntityReader reader = inputFormat.createReader(new InputRowSchema( new TimestampSpec("timestamp", "iso", null), dimensionsSpecWithExclusions, ColumnsFilter.all() - ), kafkaRecordEntity, null).read(); - - - Assert.assertTrue(rows.hasNext()); - InputRow row = rows.next(); - - Assert.assertEquals(4, row.getDimensions().size()); - assertDimensionEquals(row, "metric.name", "example_gauge"); - assertDimensionEquals(row, "value", "6"); - assertDimensionEquals(row, "custom.env", "devel"); - assertDimensionEquals(row, "descriptor.foo_key", "foo_value"); - Assert.assertFalse(row.getDimensions().contains("custom.country")); - Assert.assertFalse(row.getDimensions().contains("descriptor.color")); + ), entity, null); + + entity.setEntity(new KafkaRecordEntity(consumerRecord)); + try (CloseableIterator rows = reader.read()) { + Assert.assertTrue(rows.hasNext()); + InputRow row = rows.next(); + + Assert.assertEquals(4, row.getDimensions().size()); + assertDimensionEquals(row, "metric.name", "example_gauge"); + assertDimensionEquals(row, "value", "6"); + assertDimensionEquals(row, "custom.env", "devel"); + assertDimensionEquals(row, "descriptor.foo_key", "foo_value"); + Assert.assertFalse(row.getDimensions().contains("custom.country")); + Assert.assertFalse(row.getDimensions().contains("descriptor.color")); + } } @Test public void testInvalidProtobuf() throws IOException { byte[] invalidProtobuf = new byte[] {0x00, 0x01}; - ConsumerRecord consumerRecord = new ConsumerRecord(TOPIC, PARTITION, OFFSET, TS, TSTYPE, - -1L, -1, -1, null, invalidProtobuf, HEADERS); - KafkaRecordEntity kafkaRecordEntity = new KafkaRecordEntity(consumerRecord); + ConsumerRecord consumerRecord = new ConsumerRecord<>(TOPIC, PARTITION, OFFSET, TS, TSTYPE, -1, -1, + null, invalidProtobuf, HEADERS, Optional.empty()); OpenCensusProtobufInputFormat inputFormat = new OpenCensusProtobufInputFormat("metric.name", null, "descriptor.", "custom."); - CloseableIterator rows = inputFormat.createReader(new InputRowSchema( + SettableByteEntity entity = new SettableByteEntity<>(); + InputEntityReader reader = inputFormat.createReader(new InputRowSchema( new TimestampSpec("timestamp", "iso", null), dimensionsSpec, ColumnsFilter.all() - ), kafkaRecordEntity, null).read(); + ), entity, null); - Assert.assertThrows(ParseException.class, () -> rows.hasNext()); - Assert.assertThrows(ParseException.class, () -> rows.next()); + entity.setEntity(new KafkaRecordEntity(consumerRecord)); + try (CloseableIterator rows = reader.read()) { + Assert.assertThrows(ParseException.class, () -> rows.hasNext()); + Assert.assertThrows(ParseException.class, () -> rows.next()); + } } private void assertDimensionEquals(InputRow row, String dimension, Object expected) diff --git a/extensions-contrib/opencensus-extensions/src/test/resources/log4j2.xml b/extensions-contrib/opencensus-extensions/src/test/resources/log4j2.xml new file mode 100644 index 000000000000..05a8e1d69cbe --- /dev/null +++ b/extensions-contrib/opencensus-extensions/src/test/resources/log4j2.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + diff --git a/extensions-contrib/opentelemetry-extensions/pom.xml b/extensions-contrib/opentelemetry-extensions/pom.xml index 76bd2d9c1846..75d67f621bfb 100644 --- a/extensions-contrib/opentelemetry-extensions/pom.xml +++ b/extensions-contrib/opentelemetry-extensions/pom.xml @@ -73,6 +73,12 @@ ${project.parent.version} provided + + org.apache.druid + druid-indexing-service + ${project.parent.version} + provided + junit diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufInputFormat.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufInputFormat.java index 2089c2c7d676..50029e8dfbd9 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufInputFormat.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufInputFormat.java @@ -25,6 +25,7 @@ import org.apache.druid.data.input.InputFormat; import org.apache.druid.data.input.InputRowSchema; import org.apache.druid.data.input.impl.ByteEntity; +import org.apache.druid.indexing.seekablestream.SettableByteEntity; import org.apache.druid.java.util.common.StringUtils; import java.io.File; @@ -63,9 +64,19 @@ public boolean isSplittable() @Override public InputEntityReader createReader(InputRowSchema inputRowSchema, InputEntity source, File temporaryDirectory) { + // Sampler passes a KafkaRecordEntity directly, while the normal code path wraps the same entity in a + // SettableByteEntity + SettableByteEntity settableEntity; + if (source instanceof SettableByteEntity) { + settableEntity = (SettableByteEntity) source; + } else { + SettableByteEntity wrapper = new SettableByteEntity<>(); + wrapper.setEntity((ByteEntity) source); + settableEntity = wrapper; + } return new OpenTelemetryMetricsProtobufReader( inputRowSchema.getDimensionsSpec(), - (ByteEntity) source, + settableEntity, metricDimension, valueDimension, metricAttributePrefix, diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java index 27d424848ea8..966718a7d7e5 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java @@ -34,12 +34,14 @@ import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionsSpec; +import org.apache.druid.indexing.seekablestream.SettableByteEntity; import org.apache.druid.java.util.common.CloseableIterators; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.common.parsers.CloseableIterator; import org.apache.druid.java.util.common.parsers.ParseException; import javax.annotation.Nullable; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -53,7 +55,7 @@ public class OpenTelemetryMetricsProtobufReader implements InputEntityReader { private static final Logger log = new Logger(OpenTelemetryMetricsProtobufReader.class); - private final ByteEntity source; + private final SettableByteEntity source; private final String metricDimension; private final String valueDimension; private final String metricAttributePrefix; @@ -62,7 +64,7 @@ public class OpenTelemetryMetricsProtobufReader implements InputEntityReader public OpenTelemetryMetricsProtobufReader( DimensionsSpec dimensionsSpec, - ByteEntity source, + SettableByteEntity source, String metricDimension, String valueDimension, String metricAttributePrefix, @@ -98,7 +100,12 @@ public InputRow next() List readAsList() { try { - return parseMetricsData(MetricsData.parseFrom(source.getBuffer())); + ByteBuffer buffer = source.getEntity().getBuffer(); + List rows = parseMetricsData(MetricsData.parseFrom(buffer)); + // Explicitly move the position assuming that all the remaining bytes have been consumed because the protobuf + // parser does not update the position itself + buffer.position(buffer.limit()); + return rows; } catch (InvalidProtocolBufferException e) { throw new ParseException(null, e, "Protobuf message could not be parsed"); @@ -153,7 +160,7 @@ private List parseMetric(Metric metric, Map resourceAt case HISTOGRAM: case SUMMARY: default: - log.trace("Metric type {} is not supported", metric.getDataCase()); + log.trace("Metric type %s is not supported", metric.getDataCase()); inputRows = Collections.emptyList(); } diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java index e010d486d885..ded77f1881af 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java @@ -29,6 +29,7 @@ import org.apache.druid.data.input.impl.ByteEntity; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.StringDimensionSchema; +import org.apache.druid.indexing.seekablestream.SettableByteEntity; import org.apache.druid.java.util.common.parsers.CloseableIterator; import org.apache.druid.java.util.common.parsers.ParseException; import org.junit.Assert; @@ -37,6 +38,7 @@ import org.junit.Test; import org.junit.rules.ExpectedException; +import java.io.IOException; import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -110,9 +112,11 @@ public void testSumWithAttributes() MetricsData metricsData = metricsDataBuilder.build(); + SettableByteEntity settableByteEntity = new SettableByteEntity<>(); + settableByteEntity.setEntity(new ByteEntity(metricsData.toByteArray())); CloseableIterator rows = new OpenTelemetryMetricsProtobufReader( dimensionsSpec, - new ByteEntity(metricsData.toByteArray()), + settableByteEntity, "metric.name", "raw.value", "descriptor.", @@ -145,9 +149,11 @@ public void testGaugeWithAttributes() MetricsData metricsData = metricsDataBuilder.build(); + SettableByteEntity settableByteEntity = new SettableByteEntity<>(); + settableByteEntity.setEntity(new ByteEntity(metricsData.toByteArray())); CloseableIterator rows = new OpenTelemetryMetricsProtobufReader( dimensionsSpec, - new ByteEntity(metricsData.toByteArray()), + settableByteEntity, "metric.name", "raw.value", "descriptor.", @@ -205,9 +211,11 @@ public void testBatchedMetricParse() MetricsData metricsData = metricsDataBuilder.build(); + SettableByteEntity settableByteEntity = new SettableByteEntity<>(); + settableByteEntity.setEntity(new ByteEntity(metricsData.toByteArray())); CloseableIterator rows = new OpenTelemetryMetricsProtobufReader( dimensionsSpec, - new ByteEntity(metricsData.toByteArray()), + settableByteEntity, "metric.name", "raw.value", "descriptor.", @@ -262,9 +270,11 @@ public void testDimensionSpecExclusions() "custom." + RESOURCE_ATTRIBUTE_COUNTRY )).build(); + SettableByteEntity settableByteEntity = new SettableByteEntity<>(); + settableByteEntity.setEntity(new ByteEntity(metricsData.toByteArray())); CloseableIterator rows = new OpenTelemetryMetricsProtobufReader( dimensionsSpecWithExclusions, - new ByteEntity(metricsData.toByteArray()), + settableByteEntity, "metric.name", "raw.value", "descriptor.", @@ -315,9 +325,11 @@ public void testUnsupportedValueTypes() MetricsData metricsData = metricsDataBuilder.build(); + SettableByteEntity settableByteEntity = new SettableByteEntity<>(); + settableByteEntity.setEntity(new ByteEntity(metricsData.toByteArray())); CloseableIterator rows = new OpenTelemetryMetricsProtobufReader( dimensionsSpec, - new ByteEntity(metricsData.toByteArray()), + settableByteEntity, "metric.name", "raw.value", "descriptor.", @@ -347,18 +359,24 @@ public void testUnsupportedValueTypes() public void testInvalidProtobuf() { byte[] invalidProtobuf = new byte[] {0x00, 0x01}; - CloseableIterator rows = new OpenTelemetryMetricsProtobufReader( + SettableByteEntity settableByteEntity = new SettableByteEntity<>(); + settableByteEntity.setEntity(new ByteEntity(invalidProtobuf)); + try (CloseableIterator rows = new OpenTelemetryMetricsProtobufReader( dimensionsSpec, - new ByteEntity(invalidProtobuf), + settableByteEntity, "metric.name", "raw.value", "descriptor.", "custom." - ).read(); - Assert.assertThrows(ParseException.class, () -> rows.hasNext()); - Assert.assertThrows(ParseException.class, () -> rows.next()); + ).read()) { + Assert.assertThrows(ParseException.class, () -> rows.hasNext()); + Assert.assertThrows(ParseException.class, () -> rows.next()); + } + catch (IOException e) { + // Comes from the implicit call to close. Ignore + } } - + @Test public void testInvalidMetricType() { @@ -370,9 +388,11 @@ public void testInvalidMetricType() MetricsData metricsData = metricsDataBuilder.build(); + SettableByteEntity settableByteEntity = new SettableByteEntity<>(); + settableByteEntity.setEntity(new ByteEntity(metricsData.toByteArray())); CloseableIterator rows = new OpenTelemetryMetricsProtobufReader( dimensionsSpec, - new ByteEntity(metricsData.toByteArray()), + settableByteEntity, "metric.name", "raw.value", "descriptor.", From e3d5fc2cc0df2b7153a4c248bcd60f5c3399e1e7 Mon Sep 17 00:00:00 2001 From: ConfluentTools <96149134+ConfluentTools@users.noreply.github.com> Date: Thu, 26 Jan 2023 09:54:12 -0800 Subject: [PATCH 073/114] chore: update repo semaphore project (#124) Co-authored-by: Confluent Jenkins Bot --- .semaphore/project.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.semaphore/project.yml b/.semaphore/project.yml index b5669343e76c..2f7c905344f9 100644 --- a/.semaphore/project.yml +++ b/.semaphore/project.yml @@ -1,3 +1,8 @@ +# This file is managed by ServiceBot plugin - Semaphore. The content in this file is created using a common +# template and configurations in service.yml. +# Modifications in this file will be overwritten by generated content in the nightly run. +# For more information, please refer to the page: +# https://confluentinc.atlassian.net/wiki/spaces/Foundations/pages/2871296194/Add+SemaphoreCI apiVersion: v1alpha kind: Project metadata: From e55570e0bcf5a6b99c74fe6f8c0f8c60de957469 Mon Sep 17 00:00:00 2001 From: Kamal Narayan <119908061+kamal-narayan@users.noreply.github.com> Date: Mon, 6 Feb 2023 08:31:38 -0800 Subject: [PATCH 074/114] [Metrics-4776] OpenTelemetry Extensions - Upgrade otel-proto version (#125) * Upgrade proto version * Fix names and tests - Upgrade version * Fix open census tests * Fix test name --- .../OpenCensusProtobufReaderTest.java | 24 ++++++++-------- .../OpenTelemetryMetricsProtobufReader.java | 4 +-- .../protobuf/OpenTelemetryBenchmark.java | 12 ++++---- ...penTelemetryMetricsProtobufReaderTest.java | 28 +++++++++---------- pom.xml | 2 +- 5 files changed, 35 insertions(+), 35 deletions(-) diff --git a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReaderTest.java b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReaderTest.java index d79f3fdb9f41..b089e36e2357 100644 --- a/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReaderTest.java +++ b/extensions-contrib/opencensus-extensions/src/test/java/org/apache/druid/data/input/opencensus/protobuf/OpenCensusProtobufReaderTest.java @@ -63,8 +63,8 @@ public class OpenCensusProtobufReaderTest public static final String RESOURCE_ATTRIBUTE_ENV = "env"; public static final String RESOURCE_ATTRIBUTE_VALUE_DEVEL = "devel"; - public static final String INSTRUMENTATION_LIBRARY_NAME = "mock-instr-lib"; - public static final String INSTRUMENTATION_LIBRARY_VERSION = "1.0"; + public static final String INSTRUMENTATION_SCOPE_NAME = "mock-instr-lib"; + public static final String INSTRUMENTATION_SCOPE_VERSION = "1.0"; public static final String METRIC_ATTRIBUTE_COLOR = "color"; public static final String METRIC_ATTRIBUTE_VALUE_RED = "red"; @@ -75,7 +75,7 @@ public class OpenCensusProtobufReaderTest private final MetricsData.Builder metricsDataBuilder = MetricsData.newBuilder(); private final Metric.Builder metricBuilder = metricsDataBuilder.addResourceMetricsBuilder() - .addInstrumentationLibraryMetricsBuilder() + .addScopeMetricsBuilder() .addMetricsBuilder(); private final DimensionsSpec dimensionsSpec = new DimensionsSpec(ImmutableList.of( @@ -109,10 +109,10 @@ public void setUp() metricsDataBuilder .getResourceMetricsBuilder(0) - .getInstrumentationLibraryMetricsBuilder(0) - .getInstrumentationLibraryBuilder() - .setName(INSTRUMENTATION_LIBRARY_NAME) - .setVersion(INSTRUMENTATION_LIBRARY_VERSION); + .getScopeMetricsBuilder(0) + .getScopeBuilder() + .setName(INSTRUMENTATION_SCOPE_NAME) + .setVersion(INSTRUMENTATION_SCOPE_VERSION); } @@ -215,7 +215,7 @@ public void testBatchedMetricParse() throws IOException // Create Second Metric Metric.Builder gaugeMetricBuilder = metricsDataBuilder.addResourceMetricsBuilder() - .addInstrumentationLibraryMetricsBuilder() + .addScopeMetricsBuilder() .addMetricsBuilder(); metricsDataBuilder.getResourceMetricsBuilder(1) @@ -226,10 +226,10 @@ public void testBatchedMetricParse() throws IOException .build()); metricsDataBuilder.getResourceMetricsBuilder(1) - .getInstrumentationLibraryMetricsBuilder(0) - .getInstrumentationLibraryBuilder() - .setName(INSTRUMENTATION_LIBRARY_NAME) - .setVersion(INSTRUMENTATION_LIBRARY_VERSION); + .getScopeMetricsBuilder(0) + .getScopeBuilder() + .setName(INSTRUMENTATION_SCOPE_NAME) + .setVersion(INSTRUMENTATION_SCOPE_VERSION); gaugeMetricBuilder.setName("example_gauge") .getGaugeBuilder() diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java index 966718a7d7e5..c5fa65ca28e2 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java @@ -128,9 +128,9 @@ private List parseMetricsData(final MetricsData metricsData) } }, HashMap::putAll); - return resourceMetrics.getInstrumentationLibraryMetricsList() + return resourceMetrics.getScopeMetricsList() .stream() - .flatMap(libraryMetrics -> libraryMetrics.getMetricsList() + .flatMap(scopeMetrics -> scopeMetrics.getMetricsList() .stream() .flatMap(metric -> parseMetric(metric, resourceAttributes).stream())); }) diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java index cf5b19b70e4a..0238aeccafa5 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryBenchmark.java @@ -22,11 +22,11 @@ import com.google.common.collect.ImmutableList; import io.opentelemetry.proto.common.v1.AnyValue; import io.opentelemetry.proto.common.v1.KeyValue; -import io.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics; import io.opentelemetry.proto.metrics.v1.Metric; import io.opentelemetry.proto.metrics.v1.MetricsData; import io.opentelemetry.proto.metrics.v1.NumberDataPoint; import io.opentelemetry.proto.metrics.v1.ResourceMetrics; +import io.opentelemetry.proto.metrics.v1.ScopeMetrics; import io.opentelemetry.proto.resource.v1.Resource; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRowSchema; @@ -58,7 +58,7 @@ public class OpenTelemetryBenchmark private int resourceMetricCount = 1; @Param(value = {"1"}) - private int instrumentationLibraryCount = 1; + private int instrumentationScopeCount = 1; @Param(value = {"1", "2", "4", "8" }) private int metricsCount = 1; @@ -94,12 +94,12 @@ private ByteBuffer createMetricBuffer() resourceAttributeBuilder.setValue(AnyValue.newBuilder().setStringValue("resource.label_value")); } - for (int j = 0; j < instrumentationLibraryCount; j++) { - InstrumentationLibraryMetrics.Builder instrumentationLibraryMetricsBuilder = - resourceMetricsBuilder.addInstrumentationLibraryMetricsBuilder(); + for (int j = 0; j < instrumentationScopeCount; j++) { + ScopeMetrics.Builder scopeMetricsBuilder = + resourceMetricsBuilder.addScopeMetricsBuilder(); for (int k = 0; k < metricsCount; k++) { - Metric.Builder metricBuilder = instrumentationLibraryMetricsBuilder.addMetricsBuilder(); + Metric.Builder metricBuilder = scopeMetricsBuilder.addMetricsBuilder(); metricBuilder.setName("io.confluent.domain/such/good/metric/wow"); for (int l = 0; l < dataPointCount; l++) { diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java index ded77f1881af..70c60bd00dd2 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java @@ -53,8 +53,8 @@ public class OpenTelemetryMetricsProtobufReaderTest public static final String RESOURCE_ATTRIBUTE_ENV = "env"; public static final String RESOURCE_ATTRIBUTE_VALUE_DEVEL = "devel"; - public static final String INSTRUMENTATION_LIBRARY_NAME = "mock-instr-lib"; - public static final String INSTRUMENTATION_LIBRARY_VERSION = "1.0"; + public static final String INSTRUMENTATION_SCOPE_NAME = "mock-instr-lib"; + public static final String INSTRUMENTATION_SCOPE_VERSION = "1.0"; public static final String METRIC_ATTRIBUTE_COLOR = "color"; public static final String METRIC_ATTRIBUTE_VALUE_RED = "red"; @@ -65,7 +65,7 @@ public class OpenTelemetryMetricsProtobufReaderTest private final MetricsData.Builder metricsDataBuilder = MetricsData.newBuilder(); private final Metric.Builder metricBuilder = metricsDataBuilder.addResourceMetricsBuilder() - .addInstrumentationLibraryMetricsBuilder() + .addScopeMetricsBuilder() .addMetricsBuilder(); private final DimensionsSpec dimensionsSpec = new DimensionsSpec(ImmutableList.of( @@ -90,10 +90,10 @@ public void setUp() metricsDataBuilder .getResourceMetricsBuilder(0) - .getInstrumentationLibraryMetricsBuilder(0) - .getInstrumentationLibraryBuilder() - .setName(INSTRUMENTATION_LIBRARY_NAME) - .setVersion(INSTRUMENTATION_LIBRARY_VERSION); + .getScopeMetricsBuilder(0) + .getScopeBuilder() + .setName(INSTRUMENTATION_SCOPE_NAME) + .setVersion(INSTRUMENTATION_SCOPE_VERSION); } @@ -184,7 +184,7 @@ public void testBatchedMetricParse() // Create Second Metric Metric.Builder gaugeMetricBuilder = metricsDataBuilder.addResourceMetricsBuilder() - .addInstrumentationLibraryMetricsBuilder() + .addScopeMetricsBuilder() .addMetricsBuilder(); metricsDataBuilder.getResourceMetricsBuilder(1) @@ -195,10 +195,10 @@ public void testBatchedMetricParse() .build()); metricsDataBuilder.getResourceMetricsBuilder(1) - .getInstrumentationLibraryMetricsBuilder(0) - .getInstrumentationLibraryBuilder() - .setName(INSTRUMENTATION_LIBRARY_NAME) - .setVersion(INSTRUMENTATION_LIBRARY_VERSION); + .getScopeMetricsBuilder(0) + .getScopeBuilder() + .setName(INSTRUMENTATION_SCOPE_NAME) + .setVersion(INSTRUMENTATION_SCOPE_VERSION); gaugeMetricBuilder.setName("example_gauge") .getGaugeBuilder() @@ -381,8 +381,8 @@ public void testInvalidProtobuf() public void testInvalidMetricType() { metricBuilder - .setName("deprecated_intsum") - .getIntSumBuilder() + .setName("unsupported_histogram_metric") + .getExponentialHistogramBuilder() .addDataPointsBuilder() .setTimeUnixNano(TIMESTAMP); diff --git a/pom.xml b/pom.xml index ef203f0ea72e..05bfe098ab25 100644 --- a/pom.xml +++ b/pom.xml @@ -133,7 +133,7 @@ maven.org Maven Central Repository https://repo1.maven.org/maven2/ - 0.11.0-alpha + 0.19.0-alpha 3 From f94d116f8112db4f2b15006915f569b5e318f193 Mon Sep 17 00:00:00 2001 From: David Steere Date: Mon, 13 Feb 2023 12:46:33 +0000 Subject: [PATCH 075/114] Move to Java 17 (#128) * bumping version of java to 17 for semaphore test run * bumping java version to 17 as per https://github.com/confluentinc/druid/pull/127/files * After speaking with Xavier, made these changes --- .semaphore/semaphore.yml | 4 ++-- distribution/pom.xml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 768e57be0b28..2d5efb98f715 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -28,7 +28,7 @@ blocks: prologue: commands: - echo $SEMAPHORE_WORKFLOW_ID - - sem-version java 11 + - sem-version java 17 - checkout jobs: - name: "Install" @@ -45,7 +45,7 @@ blocks: prologue: commands: - echo $SEMAPHORE_WORKFLOW_ID - - sem-version java 11 + - sem-version java 17 - checkout - cache restore jobs: diff --git a/distribution/pom.xml b/distribution/pom.xml index eeba70f016c6..8ed888df1e0e 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -50,7 +50,7 @@ --clean - 11 + 17 From d2d6b14f054069c91c29c1d532725c36995db31b Mon Sep 17 00:00:00 2001 From: David Steere Date: Fri, 24 Feb 2023 11:18:46 +0000 Subject: [PATCH 076/114] Trying to add required flags to run druid using java 17 (#130) --- distribution/docker/druid.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/distribution/docker/druid.sh b/distribution/docker/druid.sh index 1bc2f4f071ab..a13aca2ad528 100755 --- a/distribution/docker/druid.sh +++ b/distribution/docker/druid.sh @@ -167,6 +167,32 @@ if [ -n "$DRUID_MAXNEWSIZE" ]; then setJavaKey ${SERVICE} -XX:MaxNewSize -XX:Max if [ -n "$DRUID_NEWSIZE" ]; then setJavaKey ${SERVICE} -XX:NewSize -XX:NewSize=${DRUID_NEWSIZE}; fi if [ -n "$DRUID_MAXDIRECTMEMORYSIZE" ]; then setJavaKey ${SERVICE} -XX:MaxDirectMemorySize -XX:MaxDirectMemorySize=${DRUID_MAXDIRECTMEMORYSIZE}; fi +# If java version is 17 or greater, add command line args to enable class loading via Guice +JAVA_MAJOR="$(java -version 2>&1 | sed -n -E 's/.* version "([^."]*).*/\1/p')" + +if [ "$JAVA_MAJOR" != "" ] && [ "$JAVA_MAJOR" -ge "17" ] +then + # Must disable strong encapsulation for certain packages on Java 17. + JAVA_OPTS="--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED \ + --add-exports=java.base/jdk.internal.perf=ALL-UNNAMED \ + --add-exports=java.base/jdk.internal.ref=ALL-UNNAMED \ + --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED \ + --add-opens=java.base/java.nio=ALL-UNNAMED \ + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \ + --add-opens=java.base/java.io=ALL-UNNAMED \ + --add-opens=java.base/java.lang=ALL-UNNAMED \ + --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED \ + $JAVA_OPTS" +elif [ "$JAVA_MAJOR" != "" ] && [ "$JAVA_MAJOR" -ge "11" ] +then + # Parameters below are required to use datasketches-memory as a library + JAVA_OPTS="$JAVA_OPTS \ + --add-exports=java.base/jdk.internal.misc=ALL-UNNAMED \ + --add-exports=java.base/jdk.internal.ref=ALL-UNNAMED \ + --add-opens=java.base/java.nio=ALL-UNNAMED \ + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED" +fi + # Combine options from jvm.config and those given as JAVA_OPTS # If a value is specified in both then JAVA_OPTS will take precedence when using OpenJDK # However this behavior is not part of the spec and is thus implementation specific From 3f72b500bea64dc2a8166d8dcbdbe9065d5bef7a Mon Sep 17 00:00:00 2001 From: Konstantine Karantasis Date: Mon, 17 Apr 2023 16:27:04 -0700 Subject: [PATCH 077/114] Use apache-jar-resource-bundle:1.5 instead of 1.5-SNAPSHOT (#14054) (#131) Co-authored-by: Tejaswini Bandlamudi <96047043+tejaswini-imply@users.noreply.github.com> From 297711408b8793e4535e9d9c4e4dd0979a8d098e Mon Sep 17 00:00:00 2001 From: Ghazanfar-CFLT Date: Tue, 30 May 2023 00:01:38 +0530 Subject: [PATCH 078/114] update parent pom version for Confluent extensions --- extensions-contrib/confluent-extensions/pom.xml | 2 +- extensions-contrib/opencensus-extensions/pom.xml | 2 +- extensions-contrib/opentelemetry-extensions/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions-contrib/confluent-extensions/pom.xml b/extensions-contrib/confluent-extensions/pom.xml index e51a72cd986b..90dbecd53a49 100644 --- a/extensions-contrib/confluent-extensions/pom.xml +++ b/extensions-contrib/confluent-extensions/pom.xml @@ -17,7 +17,7 @@ druid org.apache.druid - 24.0.2 + 25.0.0 ../../pom.xml diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml index 2d0d773c7962..0552675a8a21 100644 --- a/extensions-contrib/opencensus-extensions/pom.xml +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -31,7 +31,7 @@ druid org.apache.druid - 24.0.2 + 25.0.0 ../../pom.xml diff --git a/extensions-contrib/opentelemetry-extensions/pom.xml b/extensions-contrib/opentelemetry-extensions/pom.xml index 75d67f621bfb..971d2d9983f9 100644 --- a/extensions-contrib/opentelemetry-extensions/pom.xml +++ b/extensions-contrib/opentelemetry-extensions/pom.xml @@ -30,7 +30,7 @@ druid org.apache.druid - 24.0.2 + 25.0.0 ../../pom.xml From 15d04499c612fa4e0453c2e85778d71d9f7b09a5 Mon Sep 17 00:00:00 2001 From: Ghazanfar-CFLT Date: Tue, 30 May 2023 13:00:57 +0530 Subject: [PATCH 079/114] Fix CI/CD while upgrading to Druid 25.0.0 From f16f72ca0dba87092bdccd168cd065cfbf59d5c6 Mon Sep 17 00:00:00 2001 From: Ghazanfar-CFLT Date: Tue, 30 May 2023 16:48:56 +0530 Subject: [PATCH 080/114] Fix jest and prettify checks --- web-console/README.md | 5 ++++- web-console/console-config.js | 10 +++++----- web-console/src/ace-modes/hjson.js | 12 ++++++------ 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/web-console/README.md b/web-console/README.md index 4e16369d3436..ff5012344bd4 100644 --- a/web-console/README.md +++ b/web-console/README.md @@ -46,6 +46,7 @@ The console relies on [eslint](https://eslint.org) (and various plugins), [sass- #### Configuring WebStorm - **Preferences | Languages & Frameworks | JavaScript | Code Quality Tools | ESLint** + - Select "Automatic ESLint Configuration" - Check "Run eslint --fix on save" @@ -55,6 +56,7 @@ The console relies on [eslint](https://eslint.org) (and various plugins), [sass- - Check "On save" #### Configuring VS Code + - Install `dbaeumer.vscode-eslint` extension - Install `esbenp.prettier-vscode` extension - Open User Settings (JSON) and set the following: @@ -67,10 +69,11 @@ The console relies on [eslint](https://eslint.org) (and various plugins), [sass- ``` #### Auto-fixing manually + It is also possible to auto-fix and format code without making IDE changes by running the following script: - `npm run autofix` — run code linters and formatter - + You could also run fixers individually: - `npm run eslint-fix` — run code linter and fix issues diff --git a/web-console/console-config.js b/web-console/console-config.js index a1a4de01e9b2..e7e34c136c00 100644 --- a/web-console/console-config.js +++ b/web-console/console-config.js @@ -19,9 +19,9 @@ window.consoleConfig = { exampleManifestsUrl: 'https://druid.apache.org/data/example-manifests-v2.tsv', /* future configs may go here */ - "defaultQueryContext": { - "priority": -1, - "timeout": 30000, - "lane": "console" - } + defaultQueryContext: { + priority: -1, + timeout: 30000, + lane: 'console', + }, }; diff --git a/web-console/src/ace-modes/hjson.js b/web-console/src/ace-modes/hjson.js index 17142c89e597..084017217bd6 100644 --- a/web-console/src/ace-modes/hjson.js +++ b/web-console/src/ace-modes/hjson.js @@ -25,15 +25,15 @@ ace.define( 'ace/mode/hjson_highlight_rules', ['require', 'exports', 'module', 'ace/lib/oop', 'ace/mode/text_highlight_rules'], - function(acequire, exports, module) { + function (acequire, exports, module) { 'use strict'; var oop = acequire('../lib/oop'); var TextHighlightRules = acequire('./text_highlight_rules').TextHighlightRules; - var HjsonHighlightRules = function() { + var HjsonHighlightRules = function () { this.$rules = { - start: [ + 'start': [ { include: '#comments', }, @@ -277,19 +277,19 @@ ace.define( 'ace/mode/text', 'ace/mode/hjson_highlight_rules', ], - function(acequire, exports, module) { + function (acequire, exports, module) { 'use strict'; var oop = acequire('../lib/oop'); var TextMode = acequire('./text').Mode; var HjsonHighlightRules = acequire('./hjson_highlight_rules').HjsonHighlightRules; - var Mode = function() { + var Mode = function () { this.HighlightRules = HjsonHighlightRules; }; oop.inherits(Mode, TextMode); - (function() { + (function () { this.lineCommentStart = '//'; this.blockComment = { start: '/*', end: '*/' }; this.$id = 'ace/mode/hjson'; From db8bfbf954d9e1c1a2dc6d3c15850cc9826fffda Mon Sep 17 00:00:00 2001 From: Harini Rajendran Date: Thu, 8 Jun 2023 08:47:38 -0500 Subject: [PATCH 081/114] Adding SegmentMetadataEvent and publishing them via KafkaEmitter (#14281) (#139) (cherry picked from commit 4ff6026d30e4da53dc0e37bc2279d9e030773787) --- .../extensions-contrib/kafka-emitter.md | 26 +++++++++---------- .../druid/emitter/kafka/KafkaEmitter.java | 2 +- .../emitter/kafka/KafkaEmitterConfig.java | 5 ++-- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/docs/development/extensions-contrib/kafka-emitter.md b/docs/development/extensions-contrib/kafka-emitter.md index 637cfcae2bb8..eba138e004cd 100644 --- a/docs/development/extensions-contrib/kafka-emitter.md +++ b/docs/development/extensions-contrib/kafka-emitter.md @@ -36,28 +36,28 @@ to monitor the status of your Druid cluster with this extension. All the configuration parameters for the Kafka emitter are under `druid.emitter.kafka`. -| property | description | required? | default | -|----------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------|-----------------------| -| `druid.emitter.kafka.bootstrap.servers` | Comma-separated Kafka broker. (`[hostname:port],[hostname:port]...`) | yes | none | -| `druid.emitter.kafka.event.types` | Comma-separated event types.
Choices: alerts, metrics, requests, segmentMetadata | no | ["metrics", "alerts"] | -| `druid.emitter.kafka.metric.topic` | Kafka topic name for emitter's target to emit service metric. If `event.types` contains `metrics`, this field cannot be left empty | no | none | -| `druid.emitter.kafka.alert.topic` | Kafka topic name for emitter's target to emit alert. If `event.types` contains `alerts`, this field cannot be left empty | no | none | -| `druid.emitter.kafka.request.topic` | Kafka topic name for emitter's target to emit request logs. If `event.types` contains `requests`, this field cannot be left empty | no | none | -| `druid.emitter.kafka.segmentMetadata.topic` | Kafka topic name for emitter's target to emit segments related metadata. If `event.types` contains `segmentMetadata`, this field cannot be left empty | no | none | +| Property | Description | Required | Default | +|----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|-----------|-----------------------| +| `druid.emitter.kafka.bootstrap.servers` | Comma-separated Kafka broker. (`[hostname:port],[hostname:port]...`) | yes | none | +| `druid.emitter.kafka.event.types` | Comma-separated event types.
Supported types are `alerts`, `metrics`, `requests`, and `segment_metadata`. | no | `["metrics", "alerts"]` | +| `druid.emitter.kafka.metric.topic` | Kafka topic name for emitter's target to emit service metrics. If `event.types` contains `metrics`, this field cannot be empty. | no | none | +| `druid.emitter.kafka.alert.topic` | Kafka topic name for emitter's target to emit alerts. If `event.types` contains `alerts`, this field cannot empty. | no | none | +| `druid.emitter.kafka.request.topic` | Kafka topic name for emitter's target to emit request logs. If `event.types` contains `requests`, this field cannot be empty. | no | none | +| `druid.emitter.kafka.segmentMetadata.topic` | Kafka topic name for emitter's target to emit segment metadata. If `event.types` contains `segment_metadata`, this field cannot be empty. | no | none | | `druid.emitter.kafka.segmentMetadata.topic.format` | Format in which segment related metadata will be emitted.
Choices: json, protobuf.
If set to `protobuf`, then segment metadata is emitted in `DruidSegmentEvent.proto` format | no | json | -| `druid.emitter.kafka.producer.config` | JSON formatted configuration which user want to set additional properties to Kafka producer. | no | none | -| `druid.emitter.kafka.clusterName` | Optional value to specify name of your druid cluster. It can help make groups in your monitoring environment. | no | none | +| `druid.emitter.kafka.producer.config` | JSON configuration to set additional properties to Kafka producer. | no | none | +| `druid.emitter.kafka.clusterName` | Optional value to specify the name of your Druid cluster. It can help make groups in your monitoring environment. | no | none | ### Example ``` druid.emitter.kafka.bootstrap.servers=hostname1:9092,hostname2:9092 -druid.emitter.kafka.event.types=["alerts", "requests", "segmentMetadata"] +druid.emitter.kafka.event.types=["metrics", alerts", "requests", "segment_metadata"] +druid.emitter.kafka.metric.topic=druid-metric druid.emitter.kafka.alert.topic=druid-alert druid.emitter.kafka.request.topic=druid-request-logs druid.emitter.kafka.segmentMetadata.topic=druid-segment-metadata -druid.emitter.kafka.segmentMetadata.topic.format=protobuf +druid.emitter.kafka.segmentMetadata.topic.format=protobuf druid.emitter.kafka.producer.config={"max.block.ms":10000} ``` Whenever `druid.emitter.kafka.segmentMetadata.topic.format` field is updated, it is recommended to also update `druid.emitter.kafka.segmentMetadata.topic` to avoid the same topic from getting polluted with different formats of segment metadata. - diff --git a/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitter.java b/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitter.java index 8bd4ecc019f5..54f563eb3921 100644 --- a/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitter.java +++ b/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitter.java @@ -142,7 +142,7 @@ public void start() scheduler.schedule(this::sendSegmentMetadataToKafka, sendInterval, TimeUnit.SECONDS); } scheduler.scheduleWithFixedDelay(() -> { - log.info("Message lost counter: metricLost=[%d], alertLost=[%d], requestLost=[%d], invalidLost=[%d] segmentMetadataLost=[%d]", + log.info("Message lost counter: metricLost=[%d], alertLost=[%d], requestLost=[%d], segmentMetadataLost=[%d], invalidLost=[%d]", metricLost.get(), alertLost.get(), requestLost.get(), diff --git a/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitterConfig.java b/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitterConfig.java index 809c7b4d79ca..09deffcc81a7 100644 --- a/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitterConfig.java +++ b/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitterConfig.java @@ -62,8 +62,6 @@ public static EventType fromString(String name) } } - public static final Set DEFAULT_EVENT_TYPES = ImmutableSet.of(EventType.ALERTS, EventType.METRICS); - public enum SegmentMetadataTopicFormat { JSON, @@ -82,6 +80,9 @@ public static SegmentMetadataTopicFormat fromString(String name) return valueOf(StringUtils.toUpperCase(name)); } } + + public static final Set DEFAULT_EVENT_TYPES = ImmutableSet.of(EventType.ALERTS, EventType.METRICS); + @JsonProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG) private final String bootstrapServers; @Nullable @JsonProperty("event.types") From 317b8ccd988a6ed2475e71be60e2e3cda4f4a470 Mon Sep 17 00:00:00 2001 From: Parth Agrawal <98726675+pagrawal10@users.noreply.github.com> Date: Fri, 7 Jul 2023 11:09:47 +0530 Subject: [PATCH 082/114] Downgrade busybox version to fix k8s IT (#14518) (#143) Co-authored-by: Rishabh Singh <6513075+findingrish@users.noreply.github.com> --- distribution/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/docker/Dockerfile b/distribution/docker/Dockerfile index 9e619f704e8c..ab50e99e7ce7 100644 --- a/distribution/docker/Dockerfile +++ b/distribution/docker/Dockerfile @@ -50,7 +50,7 @@ RUN if [ "$TARGETARCH" = "arm64" ]; then \ echo "Downloading bash-static from ${BASH_URL}" \ && wget ${BASH_URL} -O /bin/bash -FROM busybox:1.35.0-glibc as busybox +FROM busybox:1.34.1-glibc as busybox FROM gcr.io/distroless/java$JDK_VERSION-debian12 LABEL maintainer="Apache Druid Developers " From 8ebbde22c7b25e02bf637ad90dc1a48dfd059d29 Mon Sep 17 00:00:00 2001 From: Parth Agrawal <98726675+pagrawal10@users.noreply.github.com> Date: Mon, 10 Jul 2023 10:28:01 +0530 Subject: [PATCH 083/114] Passing TARGETARCH in build_args to Docker build (#144) * Downgrade busybox version to fix k8s IT (#14518) * Add TargetArch needed in distribution/Dockerfile * Fix linting --------- Co-authored-by: Rishabh Singh <6513075+findingrish@users.noreply.github.com> --- distribution/pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/distribution/pom.xml b/distribution/pom.xml index 8ed888df1e0e..cccfc58aac2e 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -51,6 +51,7 @@ --clean 17 + amd64 @@ -611,6 +612,7 @@ ${project.version} ${docker.jdk.version} + ${targetarch} From a6932fb01cb940376b8e7b7d20a91e4c438b3734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Wed, 19 Jul 2023 16:27:40 +0200 Subject: [PATCH 084/114] remove docker-maven-plugin and Dockerfile customizations - remove our custom profile to build using dockerfile-maven-plugin, since that plugin is no longer maintained. - remove our custom Dockerfile patches since we can now use the BUILD_FROM_SOURCE argument to decide if we want to build the tarball outside of docker. --- distribution/docker/Dockerfile | 32 ++++++++++++++++++----- distribution/pom.xml | 47 ---------------------------------- 2 files changed, 25 insertions(+), 54 deletions(-) diff --git a/distribution/docker/Dockerfile b/distribution/docker/Dockerfile index ab50e99e7ce7..60779aa0128f 100644 --- a/distribution/docker/Dockerfile +++ b/distribution/docker/Dockerfile @@ -22,16 +22,34 @@ FROM alpine as extractor ARG VERSION +# The platform is explicitly specified as x64 to build the Druid distribution. +# This is because it's not able to build the distribution on arm64 due to dependency problem of web-console. See: https://github.com/apache/druid/issues/13012 +# Since only java jars are shipped in the final image, it's OK to build the distribution on x64. +# Once the web-console dependency problem is resolved, we can remove the --platform directive. +FROM --platform=linux/amd64 maven:3.8.6-jdk-11-slim as builder + +# Rebuild from source in this stage +# This can be unset if the tarball was already built outside of Docker +ARG BUILD_FROM_SOURCE="true" RUN export DEBIAN_FRONTEND=noninteractive \ && apt-get -qq update \ && apt-get -qq -y install --no-install-recommends python3 python3-yaml COPY . /src WORKDIR /src -COPY ./target/apache-druid-${VERSION}-bin.tar.gz . - -RUN tar -zxf /src/apache-druid-${VERSION}-bin.tar.gz -C /opt \ - && ln -s /opt/apache-druid-${VERSION} /opt/druid +RUN --mount=type=cache,target=/root/.m2 if [ "$BUILD_FROM_SOURCE" = "true" ]; then \ + mvn -B -ff -q dependency:go-offline \ + install \ + -Pdist,bundle-contrib-exts \ + -Pskip-static-checks,skip-tests \ + -Dmaven.javadoc.skip=true \ + ; fi + +RUN --mount=type=cache,target=/root/.m2 VERSION=$(mvn -B -q org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate \ + -Dexpression=project.version -DforceStdout=true \ + ) \ + && tar -zxf ./distribution/target/apache-druid-${VERSION}-bin.tar.gz -C /opt \ + && mv /opt/apache-druid-${VERSION} /opt/druid FROM alpine:3 as bash-static ARG TARGETARCH @@ -63,9 +81,9 @@ RUN addgroup -S -g 1000 druid \ && adduser -S -u 1000 -D -H -h /opt/druid -s /bin/sh -g '' -G druid druid -COPY --chown=druid:druid --from=extractor /opt /opt -COPY ./docker/druid.sh /druid.sh -COPY ./docker/peon.sh /peon.sh +COPY --chown=druid:druid --from=builder /opt /opt +COPY distribution/docker/druid.sh /druid.sh +COPY distribution/docker/peon.sh /peon.sh # create necessary directories which could be mounted as volume # /opt/druid/var is used to keep individual files(e.g. log) of each Druid service diff --git a/distribution/pom.xml b/distribution/pom.xml index cccfc58aac2e..5a858b816352 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -50,8 +50,6 @@ --clean - 17 - amd64 @@ -574,50 +572,5 @@ - - docker - - - - com.spotify - dockerfile-maven-plugin - 1.4.10 - - - tag-latest - - build - tag - - - docker.io/apache/druid - latest - docker/Dockerfile - - - - tag-version - - build - tag - - - docker.io/apache/druid - ${project.version} - docker/Dockerfile - - - - - - ${project.version} - ${docker.jdk.version} - ${targetarch} - - - - - - From 8a628e37a6ce64f9eba43769065af2a12d917a64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Fri, 21 Jul 2023 13:23:42 +0200 Subject: [PATCH 085/114] Revert "Trying to add required flags to run druid using java 17 (#130)" (#147) This reverts our custom patch from commit 7cf2de4081bc9196471436654bf5ebe268611e80. The necessary Java 17 exports are now included as part of 25.0.0 in https://github.com/confluentinc/druid/blob/25.0.0-confluent/examples/bin/run-java#L27-L56 which is now called by the druid.sh docker startup script as well. The exports for java.base/jdk.internal.perf=ALL-UNNAMED are no longer needed since https://github.com/apache/druid/pull/12481#discussion_r859396192 --- distribution/docker/druid.sh | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/distribution/docker/druid.sh b/distribution/docker/druid.sh index a13aca2ad528..1bc2f4f071ab 100755 --- a/distribution/docker/druid.sh +++ b/distribution/docker/druid.sh @@ -167,32 +167,6 @@ if [ -n "$DRUID_MAXNEWSIZE" ]; then setJavaKey ${SERVICE} -XX:MaxNewSize -XX:Max if [ -n "$DRUID_NEWSIZE" ]; then setJavaKey ${SERVICE} -XX:NewSize -XX:NewSize=${DRUID_NEWSIZE}; fi if [ -n "$DRUID_MAXDIRECTMEMORYSIZE" ]; then setJavaKey ${SERVICE} -XX:MaxDirectMemorySize -XX:MaxDirectMemorySize=${DRUID_MAXDIRECTMEMORYSIZE}; fi -# If java version is 17 or greater, add command line args to enable class loading via Guice -JAVA_MAJOR="$(java -version 2>&1 | sed -n -E 's/.* version "([^."]*).*/\1/p')" - -if [ "$JAVA_MAJOR" != "" ] && [ "$JAVA_MAJOR" -ge "17" ] -then - # Must disable strong encapsulation for certain packages on Java 17. - JAVA_OPTS="--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED \ - --add-exports=java.base/jdk.internal.perf=ALL-UNNAMED \ - --add-exports=java.base/jdk.internal.ref=ALL-UNNAMED \ - --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED \ - --add-opens=java.base/java.nio=ALL-UNNAMED \ - --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \ - --add-opens=java.base/java.io=ALL-UNNAMED \ - --add-opens=java.base/java.lang=ALL-UNNAMED \ - --add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED \ - $JAVA_OPTS" -elif [ "$JAVA_MAJOR" != "" ] && [ "$JAVA_MAJOR" -ge "11" ] -then - # Parameters below are required to use datasketches-memory as a library - JAVA_OPTS="$JAVA_OPTS \ - --add-exports=java.base/jdk.internal.misc=ALL-UNNAMED \ - --add-exports=java.base/jdk.internal.ref=ALL-UNNAMED \ - --add-opens=java.base/java.nio=ALL-UNNAMED \ - --add-opens=java.base/sun.nio.ch=ALL-UNNAMED" -fi - # Combine options from jvm.config and those given as JAVA_OPTS # If a value is specified in both then JAVA_OPTS will take precedence when using OpenJDK # However this behavior is not part of the spec and is thus implementation specific From 92d2402bd6f322109266ccdbd1175a0feb7e1688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20L=C3=A9aut=C3=A9?= Date: Mon, 24 Jul 2023 14:27:03 +0200 Subject: [PATCH 086/114] removing use of semaphore cache as the public semaphore will not have cache (#145) (#148) * utilize workflow level caching to publish the built artifacts to the tests. otherwise turn off all caching of .m2 etc * remove .m2/settings.xml to ensure build passes without internal artifact store --------- Co-authored-by: Jeremy Kuhnash <111304461+jkuhnashconfluent@users.noreply.github.com> --- .semaphore/semaphore.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 2d5efb98f715..9888b2175bdd 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -3,7 +3,8 @@ name: Apache Druid agent: machine: type: s1-prod-ubuntu20-04-amd64-1 - +execution_time_limit: + hours: 3 blocks: - name: "Install" task: @@ -33,11 +34,16 @@ blocks: jobs: - name: "Install" commands: - - cache restore + # This is a change meant to validate semaphore public builds + # so thus removing configurations for Confluent's internal CodeArtifact + - rm ~/.m2/settings.xml - > MAVEN_OPTS="${MAVEN_OPTS} -Xmx3000m" ${MVN} clean install -q -ff ${MAVEN_SKIP} ${MAVEN_SKIP_TESTS} -T1C - - cache store + # downstream tests depend on artifacts installed by mvn install into .m2 + # also cache target to avoid the cost of recompiling tests + - tar zcf cache-post-install.tgz .m2 target + - artifact push workflow cache-post-install.tgz - name: "Tests" task: @@ -47,7 +53,11 @@ blocks: - echo $SEMAPHORE_WORKFLOW_ID - sem-version java 17 - checkout - - cache restore + - artifact pull workflow cache-post-install.tgz + - tar zxf cache-post-install.tgz + # This is a change meant to validate semaphore public builds + # so thus removing configurations for Confluent's internal CodeArtifact + - rm ~/.m2/settings.xml jobs: - name: "animal sniffer checks" commands: From 30cf4508202b173d37d19b116baa034b8da50e56 Mon Sep 17 00:00:00 2001 From: Hardik Bajaj <58038410+hardikbajaj@users.noreply.github.com> Date: Tue, 25 Jul 2023 22:51:43 +0530 Subject: [PATCH 087/114] OBSDATA-1365: add support for debian based base images (#149) * Debeian based base image upgrade * updated suggestions * Update Dockerfile * minor correction --------- --- distribution/docker/Dockerfile | 67 +++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 25 deletions(-) diff --git a/distribution/docker/Dockerfile b/distribution/docker/Dockerfile index 60779aa0128f..146f4dd9574e 100644 --- a/distribution/docker/Dockerfile +++ b/distribution/docker/Dockerfile @@ -18,9 +18,7 @@ # ARG JDK_VERSION=17 -FROM alpine as extractor - -ARG VERSION +ARG BASE_IMAGE=gcr.io/distroless/java$JDK_VERSION-debian12 # The platform is explicitly specified as x64 to build the Druid distribution. # This is because it's not able to build the distribution on arm64 due to dependency problem of web-console. See: https://github.com/apache/druid/issues/13012 @@ -31,6 +29,7 @@ FROM --platform=linux/amd64 maven:3.8.6-jdk-11-slim as builder # Rebuild from source in this stage # This can be unset if the tarball was already built outside of Docker ARG BUILD_FROM_SOURCE="true" + RUN export DEBIAN_FRONTEND=noninteractive \ && apt-get -qq update \ && apt-get -qq -y install --no-install-recommends python3 python3-yaml @@ -51,35 +50,46 @@ RUN --mount=type=cache,target=/root/.m2 VERSION=$(mvn -B -q org.apache.maven.plu && tar -zxf ./distribution/target/apache-druid-${VERSION}-bin.tar.gz -C /opt \ && mv /opt/apache-druid-${VERSION} /opt/druid -FROM alpine:3 as bash-static +FROM busybox:1.34.1-glibc as busybox + +FROM $BASE_IMAGE + +LABEL maintainer="Apache Druid Developers " + +COPY --from=busybox /bin/busybox /busybox/busybox +RUN ["/busybox/busybox", "sh", "-c", "if [ ! -x \"$(command -v bash)\" ]; then \ + /busybox/busybox --install /bin; \ + else \ + rm /busybox/busybox; \ + fi;"] +# Predefined builtin arg, see: https://docs.docker.com/engine/reference/builder/#automatic-platform-args-in-the-global-scope ARG TARGETARCH + # # Download bash-static binary to execute scripts that require bash. # Although bash-static supports multiple platforms, but there's no need for us to support all those platform, amd64 and arm64 are enough. # ARG BASH_URL_BASE="https://github.com/robxu9/bash-static/releases/download/5.1.016-1.2.3" -RUN if [ "$TARGETARCH" = "arm64" ]; then \ - BASH_URL="${BASH_URL_BASE}/bash-linux-aarch64" ; \ - elif [ "$TARGETARCH" = "amd64" ]; then \ - BASH_URL="${BASH_URL_BASE}/bash-linux-x86_64" ; \ +RUN if [ ! -x "$(command -v bash)" ]; then \ + if [ "$TARGETARCH" = "arm64" ]; then \ + BASH_URL="${BASH_URL_BASE}/bash-linux-aarch64" ; \ + elif [ "$TARGETARCH" = "amd64" ]; then \ + BASH_URL="${BASH_URL_BASE}/bash-linux-x86_64" ; \ + else \ + echo "Unsupported architecture ($TARGETARCH)" && exit 1; \ + fi; \ + echo "Downloading bash-static from ${BASH_URL}" \ + && wget ${BASH_URL} -O /bin/bash \ + && chmod 755 /bin/bash; \ + fi; + +RUN if [ ! -x "$(command -v useradd)" ]; then \ + addgroup -S -g 1000 druid \ + && adduser -S -u 1000 -D -H -h /opt/druid -s /bin/sh -g '' -G druid druid; \ else \ - echo "Unsupported architecture ($TARGETARCH)" && exit 1; \ - fi; \ - echo "Downloading bash-static from ${BASH_URL}" \ - && wget ${BASH_URL} -O /bin/bash - -FROM busybox:1.34.1-glibc as busybox - -FROM gcr.io/distroless/java$JDK_VERSION-debian12 -LABEL maintainer="Apache Druid Developers " - -COPY --from=busybox /bin/busybox /busybox/busybox -RUN ["/busybox/busybox", "--install", "/bin"] - - -RUN addgroup -S -g 1000 druid \ - && adduser -S -u 1000 -D -H -h /opt/druid -s /bin/sh -g '' -G druid druid - + groupadd --system --gid 1000 druid \ + && useradd --system --uid 1000 -M --home /opt/druid --shell /bin/sh -c '' --gid 1000 druid; \ + fi; COPY --chown=druid:druid --from=builder /opt /opt COPY distribution/docker/druid.sh /druid.sh @@ -92,6 +102,13 @@ RUN mkdir /opt/druid/var /opt/shared \ && chown druid:druid /opt/druid/var /opt/shared \ && chmod 775 /opt/druid/var /opt/shared +# Install iproute2 to get the ip command needed to set config of druid.host IP address +# Command needed in druid.sh Line 140; +RUN if [ ! -x "$(command -v ip)" ]; then \ + apt update \ + && apt install -y iproute2; \ + fi; + USER druid VOLUME /opt/druid/var WORKDIR /opt/druid From d5cf05b8e160f8364fc1c61cd5bd67545c518fd7 Mon Sep 17 00:00:00 2001 From: Hardik Bajaj <58038410+hardikbajaj@users.noreply.github.com> Date: Fri, 11 Aug 2023 12:56:49 +0530 Subject: [PATCH 088/114] Revert "fix KafkaInputFormat with nested columns by delegating to underlying inputRow map instead of eagerly copying (#13406) (#13447)" (#155) This reverts commit 23500a4c28767c0a74b2557f8966e13720b7511b. --- .../input/kafkainput/KafkaInputReader.java | 296 ++++-------------- .../kafkainput/KafkaInputFormatTest.java | 7 +- 2 files changed, 68 insertions(+), 235 deletions(-) diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputReader.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputReader.java index 31b7cf66be19..9895f0f67432 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputReader.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputReader.java @@ -20,35 +20,34 @@ package org.apache.druid.data.input.kafkainput; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.apache.druid.data.input.InputEntityReader; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRowListPlusRawValues; import org.apache.druid.data.input.InputRowSchema; import org.apache.druid.data.input.MapBasedInputRow; -import org.apache.druid.data.input.impl.MapInputRowParser; import org.apache.druid.data.input.kafka.KafkaRecordEntity; import org.apache.druid.indexing.seekablestream.SettableByteEntity; import org.apache.druid.java.util.common.CloseableIterators; import org.apache.druid.java.util.common.Pair; +import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.common.parsers.CloseableIterator; import org.apache.druid.java.util.common.parsers.ParseException; -import org.joda.time.DateTime; import javax.annotation.Nullable; + import java.io.IOException; -import java.util.AbstractMap; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.function.Function; -import java.util.stream.Collectors; public class KafkaInputReader implements InputEntityReader { + private static final Logger log = new Logger(KafkaInputReader.class); + private final InputRowSchema inputRowSchema; private final SettableByteEntity source; private final Function headerParserSupplier; @@ -56,7 +55,6 @@ public class KafkaInputReader implements InputEntityReader private final InputEntityReader valueParser; private final String keyColumnName; private final String timestampColumnName; - private final String topicColumnName; /** * @@ -75,8 +73,7 @@ public KafkaInputReader( @Nullable Function keyParserSupplier, InputEntityReader valueParser, String keyColumnName, - String timestampColumnName, - String topicColumnName + String timestampColumnName ) { this.inputRowSchema = inputRowSchema; @@ -86,92 +83,25 @@ public KafkaInputReader( this.valueParser = valueParser; this.keyColumnName = keyColumnName; this.timestampColumnName = timestampColumnName; - this.topicColumnName = topicColumnName; - } - - @Override - public CloseableIterator read() throws IOException - { - final KafkaRecordEntity record = source.getEntity(); - final Map mergedHeaderMap = extractHeaderAndKeys(record); - - // Ignore tombstone records that have null values. - if (record.getRecord().value() != null) { - return buildBlendedRows(valueParser, mergedHeaderMap); - } else { - return CloseableIterators.withEmptyBaggage(buildInputRowsForMap(mergedHeaderMap).iterator()); - } } - @Override - public CloseableIterator sample() throws IOException + private List getFinalDimensionList(HashSet newDimensions) { - final KafkaRecordEntity record = source.getEntity(); - InputRowListPlusRawValues keysAndHeader = extractHeaderAndKeysSample(record); - if (record.getRecord().value() != null) { - return buildBlendedRowsSample(valueParser, keysAndHeader.getRawValues()); + final List schemaDimensions = inputRowSchema.getDimensionsSpec().getDimensionNames(); + if (!schemaDimensions.isEmpty()) { + return schemaDimensions; } else { - final List rows = Collections.singletonList(keysAndHeader); - return CloseableIterators.withEmptyBaggage(rows.iterator()); + return Lists.newArrayList( + Sets.difference(newDimensions, inputRowSchema.getDimensionsSpec().getDimensionExclusions()) + ); } } - private Map extractHeader(KafkaRecordEntity record) - { - final Map mergedHeaderMap = new HashMap<>(); - if (headerParserSupplier != null) { - KafkaHeaderReader headerParser = headerParserSupplier.apply(record); - List> headerList = headerParser.read(); - for (Pair ele : headerList) { - mergedHeaderMap.put(ele.lhs, ele.rhs); - } - } - - // Add kafka record timestamp to the mergelist, we will skip record timestamp if the same key exists already in - // the header list - mergedHeaderMap.putIfAbsent(timestampColumnName, record.getRecord().timestamp()); - - // Add kafka record topic to the mergelist, only if the key doesn't already exist - mergedHeaderMap.putIfAbsent(topicColumnName, record.getRecord().topic()); - - return mergedHeaderMap; - } - - private Map extractHeaderAndKeys(KafkaRecordEntity record) throws IOException - { - final Map mergedHeaderMap = extractHeader(record); - final InputEntityReader keyParser = (keyParserSupplier == null) ? null : keyParserSupplier.apply(record); - if (keyParser != null) { - try (CloseableIterator keyIterator = keyParser.read()) { - // Key currently only takes the first row and ignores the rest. - if (keyIterator.hasNext()) { - // Return type for the key parser should be of type MapBasedInputRow - // Parsers returning other types are not compatible currently. - MapBasedInputRow keyRow = (MapBasedInputRow) keyIterator.next(); - // Add the key to the mergeList only if the key string is not already present - mergedHeaderMap.putIfAbsent( - keyColumnName, - keyRow.getEvent().entrySet().stream().findFirst().get().getValue() - ); - } - } - catch (ClassCastException e) { - throw new IOException( - "Unsupported keyFormat. KafkaInputformat only supports input format that return MapBasedInputRow rows" - ); - } - } - return mergedHeaderMap; - } - - private CloseableIterator buildBlendedRows( - InputEntityReader valueParser, - Map headerKeyList - ) throws IOException + private CloseableIterator buildBlendedRows(InputEntityReader valueParser, Map headerKeyList) throws IOException { return valueParser.read().map( r -> { - final MapBasedInputRow valueRow; + MapBasedInputRow valueRow; try { // Return type for the value parser should be of type MapBasedInputRow // Parsers returning other types are not compatible currently. @@ -183,177 +113,83 @@ private CloseableIterator buildBlendedRows( "Unsupported input format in valueFormat. KafkaInputFormat only supports input format that return MapBasedInputRow rows" ); } - - final Map event = buildBlendedEventMap(valueRow.getEvent(), headerKeyList); - final HashSet newDimensions = new HashSet<>(valueRow.getDimensions()); + Map event = new HashMap<>(headerKeyList); + /* Currently we prefer payload attributes if there is a collision in names. + We can change this beahvior in later changes with a config knob. This default + behavior lets easy porting of existing inputFormats to the new one without any changes. + */ + event.putAll(valueRow.getEvent()); + + HashSet newDimensions = new HashSet(valueRow.getDimensions()); newDimensions.addAll(headerKeyList.keySet()); // Remove the dummy timestamp added in KafkaInputFormat newDimensions.remove(KafkaInputFormat.DEFAULT_AUTO_TIMESTAMP_STRING); - final DateTime timestamp = MapInputRowParser.parseTimestamp(inputRowSchema.getTimestampSpec(), event); return new MapBasedInputRow( - timestamp, - MapInputRowParser.findDimensions( - inputRowSchema.getTimestampSpec(), - inputRowSchema.getDimensionsSpec(), - newDimensions - ), + inputRowSchema.getTimestampSpec().extractTimestamp(event), + getFinalDimensionList(newDimensions), event ); } ); } - private InputRowListPlusRawValues extractHeaderAndKeysSample(KafkaRecordEntity record) throws IOException + private CloseableIterator buildRowsWithoutValuePayload(Map headerKeyList) { - Map mergedHeaderMap = extractHeader(record); + HashSet newDimensions = new HashSet(headerKeyList.keySet()); + InputRow row = new MapBasedInputRow( + inputRowSchema.getTimestampSpec().extractTimestamp(headerKeyList), + getFinalDimensionList(newDimensions), + headerKeyList + ); + List rows = Collections.singletonList(row); + return CloseableIterators.withEmptyBaggage(rows.iterator()); + } + + @Override + public CloseableIterator read() throws IOException + { + KafkaRecordEntity record = source.getEntity(); + Map mergeMap = new HashMap<>(); + if (headerParserSupplier != null) { + KafkaHeaderReader headerParser = headerParserSupplier.apply(record); + List> headerList = headerParser.read(); + for (Pair ele : headerList) { + mergeMap.put(ele.lhs, ele.rhs); + } + } + + // Add kafka record timestamp to the mergelist, we will skip record timestamp if the same key exists already in the header list + mergeMap.putIfAbsent(timestampColumnName, record.getRecord().timestamp()); + InputEntityReader keyParser = (keyParserSupplier == null) ? null : keyParserSupplier.apply(record); if (keyParser != null) { - try (CloseableIterator keyIterator = keyParser.sample()) { + try (CloseableIterator keyIterator = keyParser.read()) { // Key currently only takes the first row and ignores the rest. if (keyIterator.hasNext()) { // Return type for the key parser should be of type MapBasedInputRow // Parsers returning other types are not compatible currently. - InputRowListPlusRawValues keyRow = keyIterator.next(); + MapBasedInputRow keyRow = (MapBasedInputRow) keyIterator.next(); // Add the key to the mergeList only if the key string is not already present - mergedHeaderMap.putIfAbsent( - keyColumnName, - keyRow.getRawValues().entrySet().stream().findFirst().get().getValue() - ); - return InputRowListPlusRawValues.of(buildInputRowsForMap(mergedHeaderMap), mergedHeaderMap); + mergeMap.putIfAbsent(keyColumnName, keyRow.getEvent().entrySet().stream().findFirst().get().getValue()); } } catch (ClassCastException e) { - throw new IOException( - "Unsupported keyFormat. KafkaInputformat only supports input format that return MapBasedInputRow rows" - ); + throw new IOException("Unsupported input format in keyFormat. KafkaInputformat only supports input format that return MapBasedInputRow rows"); } } - return InputRowListPlusRawValues.of(buildInputRowsForMap(mergedHeaderMap), mergedHeaderMap); - } - - private CloseableIterator buildBlendedRowsSample( - InputEntityReader valueParser, - Map headerKeyList - ) throws IOException - { - return valueParser.sample().map( - rowAndValues -> { - if (rowAndValues.getParseException() != null) { - return rowAndValues; - } - List newInputRows = Lists.newArrayListWithCapacity(rowAndValues.getInputRows().size()); - List> newRawRows = Lists.newArrayListWithCapacity(rowAndValues.getRawValues().size()); - ParseException parseException = null; - - for (Map raw : rowAndValues.getRawValuesList()) { - newRawRows.add(buildBlendedEventMap(raw, headerKeyList)); - } - for (InputRow r : rowAndValues.getInputRows()) { - MapBasedInputRow valueRow = null; - try { - valueRow = (MapBasedInputRow) r; - } - catch (ClassCastException e) { - parseException = new ParseException( - null, - "Unsupported input format in valueFormat. KafkaInputFormat only supports input format that return MapBasedInputRow rows" - ); - } - if (valueRow != null) { - final Map event = buildBlendedEventMap(valueRow.getEvent(), headerKeyList); - final HashSet newDimensions = new HashSet<>(valueRow.getDimensions()); - newDimensions.addAll(headerKeyList.keySet()); - // Remove the dummy timestamp added in KafkaInputFormat - newDimensions.remove(KafkaInputFormat.DEFAULT_AUTO_TIMESTAMP_STRING); - newInputRows.add( - new MapBasedInputRow( - inputRowSchema.getTimestampSpec().extractTimestamp(event), - MapInputRowParser.findDimensions( - inputRowSchema.getTimestampSpec(), - inputRowSchema.getDimensionsSpec(), - newDimensions - ), - event - ) - ); - } - } - return InputRowListPlusRawValues.ofList(newRawRows, newInputRows, parseException); - } - ); - } - private List buildInputRowsForMap(Map headerKeyList) - { - return Collections.singletonList( - new MapBasedInputRow( - inputRowSchema.getTimestampSpec().extractTimestamp(headerKeyList), - MapInputRowParser.findDimensions( - inputRowSchema.getTimestampSpec(), - inputRowSchema.getDimensionsSpec(), - headerKeyList.keySet() - ), - headerKeyList - ) - ); + // Ignore tombstone records that have null values. + if (record.getRecord().value() != null) { + return buildBlendedRows(valueParser, mergeMap); + } else { + return buildRowsWithoutValuePayload(mergeMap); + } } - /** - * Builds a map that blends two {@link Map}, presenting the combined keyset of both maps, and preferring to read - * from the first map and falling back to the second map if the value is not present. - * - * This strategy is used rather than just copying the values of the keyset into a new map so that any 'flattening' - * machinery (such as {@link Map} created by {@link org.apache.druid.java.util.common.parsers.ObjectFlatteners}) is - * still in place to be lazily evaluated instead of eagerly copying. - */ - private static Map buildBlendedEventMap(Map map, Map fallback) + @Override + public CloseableIterator sample() throws IOException { - final Set keySet = new HashSet<>(fallback.keySet()); - keySet.addAll(map.keySet()); - - return new AbstractMap() - { - @Override - public Object get(Object key) - { - return map.getOrDefault((String) key, fallback.get(key)); - } - - @Override - public Set keySet() - { - return keySet; - } - - @Override - public Set> entrySet() - { - return keySet().stream() - .map( - field -> new Entry() - { - @Override - public String getKey() - { - return field; - } - - @Override - public Object getValue() - { - return get(field); - } - - @Override - public Object setValue(final Object value) - { - throw new UnsupportedOperationException(); - } - } - ) - .collect(Collectors.toCollection(LinkedHashSet::new)); - } - }; + return read().map(row -> InputRowListPlusRawValues.of(row, ((MapBasedInputRow) row).getEvent())); } -} +} \ No newline at end of file diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/data/input/kafkainput/KafkaInputFormatTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/data/input/kafkainput/KafkaInputFormatTest.java index 21a0550f53e7..0321a4da4218 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/data/input/kafkainput/KafkaInputFormatTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/data/input/kafkainput/KafkaInputFormatTest.java @@ -22,12 +22,12 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import org.apache.druid.data.input.ColumnsFilter; import org.apache.druid.data.input.InputEntityReader; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRowSchema; +import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.JsonInputFormat; import org.apache.druid.data.input.impl.TimestampSpec; @@ -250,7 +250,6 @@ public void testWithHeaderKeyAndValue() throws IOException Assert.assertEquals("4", Iterables.getOnlyElement(row.getDimension("root_baz"))); Assert.assertEquals("1", Iterables.getOnlyElement(row.getDimension("path_omg"))); Assert.assertEquals("1", Iterables.getOnlyElement(row.getDimension("jq_omg"))); - Assert.assertEquals(ImmutableMap.of("mg", 1L), row.getRaw("o")); // Header verification Assert.assertEquals("application/json", Iterables.getOnlyElement(row.getDimension("kafka.newheader.encoding"))); @@ -409,6 +408,7 @@ public byte[] value() while (iterator.hasNext()) { final InputRow row = iterator.next(); + final MapBasedInputRow mrow = (MapBasedInputRow) row; // Payload verifications // this isn't super realistic, since most of these columns are not actually defined in the dimensionSpec // but test reading them anyway since it isn't technically illegal @@ -419,7 +419,6 @@ public byte[] value() Assert.assertEquals("4", Iterables.getOnlyElement(row.getDimension("root_baz"))); Assert.assertEquals("1", Iterables.getOnlyElement(row.getDimension("path_omg"))); Assert.assertEquals("1", Iterables.getOnlyElement(row.getDimension("jq_omg"))); - Assert.assertEquals(ImmutableMap.of("mg", 1L), row.getRaw("o")); // Header verification Assert.assertEquals("application/json", Iterables.getOnlyElement(row.getDimension("kafka.newheader.encoding"))); @@ -526,7 +525,6 @@ public void testWithOutKeyAndHeaderSpecs() throws IOException Assert.assertEquals("4", Iterables.getOnlyElement(row.getDimension("root_baz"))); Assert.assertEquals("1", Iterables.getOnlyElement(row.getDimension("path_omg"))); Assert.assertEquals("1", Iterables.getOnlyElement(row.getDimension("jq_omg"))); - Assert.assertEquals(ImmutableMap.of("mg", 1L), row.getRaw("o")); numActualIterations++; } @@ -611,7 +609,6 @@ public void testWithMultipleMixedRecords() throws IOException Assert.assertEquals("4", Iterables.getOnlyElement(row.getDimension("root_baz"))); Assert.assertEquals("1", Iterables.getOnlyElement(row.getDimension("path_omg"))); Assert.assertEquals("1", Iterables.getOnlyElement(row.getDimension("jq_omg"))); - Assert.assertEquals(ImmutableMap.of("mg", 1L), row.getRaw("o")); Assert.assertEquals(String.valueOf(i), Iterables.getOnlyElement(row.getDimension("index"))); From 9aa75cf210b3ac47447b45491fbc0e2751de76dc Mon Sep 17 00:00:00 2001 From: Michael Li Date: Wed, 27 Sep 2023 10:12:58 -0700 Subject: [PATCH 089/114] Filter Out Metrics with NoRecordedValue Flag Set (#157) Metrics that contain the NoRecordedValue Flag are being written to Druid with a 0 value. We should properly handle them in the backend --- .../OpenTelemetryMetricsProtobufReader.java | 18 +++++++++++-- ...penTelemetryMetricsProtobufReaderTest.java | 27 +++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java index c5fa65ca28e2..2e0a62532b5d 100644 --- a/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java +++ b/extensions-contrib/opentelemetry-extensions/src/main/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReader.java @@ -25,6 +25,7 @@ import com.google.common.collect.Sets; import com.google.protobuf.InvalidProtocolBufferException; import io.opentelemetry.proto.common.v1.AnyValue; +import io.opentelemetry.proto.metrics.v1.DataPointFlags; import io.opentelemetry.proto.metrics.v1.Metric; import io.opentelemetry.proto.metrics.v1.MetricsData; import io.opentelemetry.proto.metrics.v1.NumberDataPoint; @@ -146,14 +147,22 @@ private List parseMetric(Metric metric, Map resourceAt inputRows = new ArrayList<>(metric.getSum().getDataPointsCount()); metric.getSum() .getDataPointsList() - .forEach(dataPoint -> inputRows.add(parseNumberDataPoint(dataPoint, resourceAttributes, metricName))); + .forEach(dataPoint -> { + if (hasRecordedValue(dataPoint)) { + inputRows.add(parseNumberDataPoint(dataPoint, resourceAttributes, metricName)); + } + }); break; } case GAUGE: { inputRows = new ArrayList<>(metric.getGauge().getDataPointsCount()); metric.getGauge() .getDataPointsList() - .forEach(dataPoint -> inputRows.add(parseNumberDataPoint(dataPoint, resourceAttributes, metricName))); + .forEach(dataPoint -> { + if (hasRecordedValue(dataPoint)) { + inputRows.add(parseNumberDataPoint(dataPoint, resourceAttributes, metricName)); + } + }); break; } // TODO Support HISTOGRAM and SUMMARY metrics @@ -167,6 +176,11 @@ private List parseMetric(Metric metric, Map resourceAt return inputRows; } + private static boolean hasRecordedValue(NumberDataPoint d) + { + return (d.getFlags() & DataPointFlags.FLAG_NO_RECORDED_VALUE_VALUE) == 0; + } + private InputRow parseNumberDataPoint(NumberDataPoint dataPoint, Map resourceAttributes, String metricName) diff --git a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java index 70c60bd00dd2..8044baf7dbe0 100644 --- a/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java +++ b/extensions-contrib/opentelemetry-extensions/src/test/java/org/apache/druid/data/input/opentelemetry/protobuf/OpenTelemetryMetricsProtobufReaderTest.java @@ -23,6 +23,7 @@ import io.opentelemetry.proto.common.v1.AnyValue; import io.opentelemetry.proto.common.v1.KeyValue; import io.opentelemetry.proto.common.v1.KeyValueList; +import io.opentelemetry.proto.metrics.v1.DataPointFlags; import io.opentelemetry.proto.metrics.v1.Metric; import io.opentelemetry.proto.metrics.v1.MetricsData; import org.apache.druid.data.input.InputRow; @@ -404,6 +405,32 @@ public void testInvalidMetricType() Assert.assertEquals(0, rowList.size()); } + @Test + public void testNoRecordedValueMetric() + { + metricBuilder.setName("example_gauge") + .getGaugeBuilder() + .addDataPointsBuilder() + .setAsInt(6) + .setFlags(DataPointFlags.FLAG_NO_RECORDED_VALUE_VALUE) + .setTimeUnixNano(TIMESTAMP); + + MetricsData metricsData = metricsDataBuilder.build(); + + SettableByteEntity settableByteEntity = new SettableByteEntity<>(); + settableByteEntity.setEntity(new ByteEntity(metricsData.toByteArray())); + CloseableIterator rows = new OpenTelemetryMetricsProtobufReader( + dimensionsSpec, + settableByteEntity, + "metric.name", + "raw.value", + "descriptor.", + "custom." + ).read(); + + Assert.assertFalse(rows.hasNext()); + } + private void assertDimensionEquals(InputRow row, String dimension, Object expected) { List values = row.getDimension(dimension); From 90b7809fc49928513d9c5bc867b7f76f97aafcab Mon Sep 17 00:00:00 2001 From: Parth Agrawal <98726675+pagrawal10@users.noreply.github.com> Date: Tue, 3 Oct 2023 16:06:05 +0530 Subject: [PATCH 090/114] memcached cache: switch to AWS elasticache-java-cluster-client and add TLS support (#14827) (#159) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR updates the library used for Memcached client to AWS Elasticache Client : https://github.com/awslabs/aws-elasticache-cluster-client-memcached-for-java This enables us to use the option of encrypting data in transit: Amazon ElastiCache for Memcached now supports encryption of data in transit For clusters running the Memcached engine, ElastiCache supports Auto Discovery—the ability for client programs to automatically identify all of the nodes in a cache cluster, and to initiate and maintain connections to all of these nodes. Benefits of Auto Discovery - Amazon ElastiCache AWS has forked spymemcached 2.12.1, and has since added all the patches included in 2.12.2 and 2.12.3 as part of the 1.2.0 release. So, this can now be considered as an equivalent drop-in replacement. GitHub - awslabs/aws-elasticache-cluster-client-memcached-for-java: Amazon ElastiCache Cluster Client for Java - enhanced library to connect to ElastiCache clusters. https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/elasticache/AmazonElastiCacheClient.html#AmazonElastiCacheClient-- How to enable TLS with Elasticache On server side: https://docs.aws.amazon.com/AmazonElastiCache/latest/mem-ug/in-transit-encryption-mc.html#in-transit-encryption-enable-existing-mc On client side: GitHub - awslabs/aws-elasticache-cluster-client-memcached-for-java: Amazon ElastiCache Cluster Client for Java - enhanced library to connect to ElastiCache clusters. From c1f42b9ef3b3731a83278e82faee252c85b0ccb9 Mon Sep 17 00:00:00 2001 From: Keerthana Srikanth Date: Thu, 9 Nov 2023 15:06:24 +0530 Subject: [PATCH 091/114] PRSP-3603 Bump org.xerial.snappy:snappy-java to latest version to address CVEs (#164) * Bump org.xerial.snappy:snappy-java from 1.1.8.4 to 1.1.10.5 * Add licenses --- licenses.yaml | 6 +++--- pom.xml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/licenses.yaml b/licenses.yaml index 0632e36f7fda..4ce3061aacb8 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -3115,7 +3115,7 @@ name: snappy-java license_category: binary module: hadoop-client license_name: Apache License version 2.0 -version: 1.0.4.1 +version: 1.1.10.5 libraries: - org.xerial.snappy: snappy-java notices: @@ -3565,7 +3565,7 @@ name: snappy-java license_category: binary module: extensions/druid-kafka-indexing-service license_name: Apache License version 2.0 -version: 1.1.8.4 +version: 1.1.10.5 libraries: - org.xerial.snappy: snappy-java @@ -4774,7 +4774,7 @@ libraries: name: snappy-java license_category: binary -version: 1.1.8.4 +version: 1.1.10.5 module: druid-ranger-security license_name: Apache License version 2.0 libraries: diff --git a/pom.xml b/pom.xml index 05bfe098ab25..00d324e7f1b1 100644 --- a/pom.xml +++ b/pom.xml @@ -814,7 +814,7 @@ org.xerial.snappy snappy-java - 1.1.10.4 + 1.1.10.5 com.google.protobuf From c0535a56e46c6616d7e2278787278c5254429035 Mon Sep 17 00:00:00 2001 From: Keerthana Srikanth Date: Tue, 14 Nov 2023 21:18:15 +0530 Subject: [PATCH 092/114] [backport] Upgrade Avro to latest version (#14440) (#162) Upgraded Avro to 1.11.1 (cherry picked from commit 72cf91fbc0ed9cc2abce91df878ab431678b12f3) Co-authored-by: Tejaswini Bandlamudi <96047043+tejaswini-imply@users.noreply.github.com> --- pom.xml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index 00d324e7f1b1..ba14853a61f2 100644 --- a/pom.xml +++ b/pom.xml @@ -79,15 +79,16 @@ 3.5.1 2.0.0 2.2.4 - 2.13.11 - 1.23.0 + 2.13.9 + 1.17.0 1.11.1 - - 1.35.0 - 4.2.0 - 2.2.0 + + 1.21.0 + 3.2.0 + 2.0.0 10.14.2.0 4.2.19 2.20.0 From 02a00d77dd5ffede667dda913bd781d085ab3543 Mon Sep 17 00:00:00 2001 From: Keerthana Srikanth Date: Wed, 15 Nov 2023 12:41:39 +0530 Subject: [PATCH 093/114] Revert "PRSP-3603 Bump org.xerial.snappy:snappy-java to latest version to address CVEs (#164)" (#166) This reverts commit 185d6559eedefee2c0a17eda7cb5750b2997710c. --- licenses.yaml | 6 +++--- pom.xml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/licenses.yaml b/licenses.yaml index 4ce3061aacb8..0632e36f7fda 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -3115,7 +3115,7 @@ name: snappy-java license_category: binary module: hadoop-client license_name: Apache License version 2.0 -version: 1.1.10.5 +version: 1.0.4.1 libraries: - org.xerial.snappy: snappy-java notices: @@ -3565,7 +3565,7 @@ name: snappy-java license_category: binary module: extensions/druid-kafka-indexing-service license_name: Apache License version 2.0 -version: 1.1.10.5 +version: 1.1.8.4 libraries: - org.xerial.snappy: snappy-java @@ -4774,7 +4774,7 @@ libraries: name: snappy-java license_category: binary -version: 1.1.10.5 +version: 1.1.8.4 module: druid-ranger-security license_name: Apache License version 2.0 libraries: diff --git a/pom.xml b/pom.xml index ba14853a61f2..6cf4ed347680 100644 --- a/pom.xml +++ b/pom.xml @@ -815,7 +815,7 @@ org.xerial.snappy snappy-java - 1.1.10.5 + 1.1.8.4 com.google.protobuf From 50ba53c858d99be446a7464f4eb45182011660e4 Mon Sep 17 00:00:00 2001 From: Keerthana Srikanth Date: Wed, 15 Nov 2023 21:28:21 +0530 Subject: [PATCH 094/114] Upgrade Avro to latest version to address CVEs (#167) --- licenses.yaml | 2 +- owasp-dependency-check-suppressions.xml | 7 ------- pom.xml | 2 +- 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/licenses.yaml b/licenses.yaml index 0632e36f7fda..2eb0e489d69a 100644 --- a/licenses.yaml +++ b/licenses.yaml @@ -3345,7 +3345,7 @@ name: Apache Avro license_category: binary module: extensions/druid-avro-extensions license_name: Apache License version 2.0 -version: 1.11.1 +version: 1.11.3 libraries: - org.apache.avro: avro - org.apache.avro: avro-mapred diff --git a/owasp-dependency-check-suppressions.xml b/owasp-dependency-check-suppressions.xml index e5cd33a5ff58..98f262c2ee2d 100644 --- a/owasp-dependency-check-suppressions.xml +++ b/owasp-dependency-check-suppressions.xml @@ -548,13 +548,6 @@ CVE-2017-3162 - - - - CVE-2021-43045 - diff --git a/pom.xml b/pom.xml index 6cf4ed347680..acc10c8ebf85 100644 --- a/pom.xml +++ b/pom.xml @@ -81,7 +81,7 @@ 2.2.4 2.13.9 1.17.0 - 1.11.1 + 1.11.3 + + dist-used + + false + + tar + + + + + + org.codehaus.mojo + exec-maven-plugin + + + generate-readme + initialize + + exec + + + ${project.basedir}/bin/build-textfile-readme.sh + + ${project.basedir}/../ + ${project.parent.version} + + + + + generate-binary-license + initialize + + exec + + + ${project.basedir}/bin/generate-binary-license.py + + ${project.parent.basedir}/licenses/APACHE2 + ${project.parent.basedir}/licenses.yaml + ${project.parent.basedir}/LICENSE.BINARY + + + + + generate-binary-notice + initialize + + exec + + + ${project.basedir}/bin/generate-binary-notice.py + + ${project.parent.basedir}/NOTICE + ${project.parent.basedir}/licenses.yaml + ${project.parent.basedir}/NOTICE.BINARY + + + + + pull-deps + package + + exec + + + ${project.parent.basedir}/examples/bin/run-java + + -classpath + + -Ddruid.extensions.loadList=[] + -Ddruid.extensions.directory=${project.build.directory}/extensions + + + -Ddruid.extensions.hadoopDependenciesDir=${project.build.directory}/hadoop-dependencies + + org.apache.druid.cli.Main + tools + pull-deps + --clean + --defaultVersion + ${project.parent.version} + -l + ${settings.localRepository} + -h + org.apache.hadoop:hadoop-client:2.8.5 + -c + org.apache.druid.extensions:druid-datasketches + -c + org.apache.druid.extensions:druid-kafka-indexing-service + -c + org.apache.druid.extensions:druid-multi-stage-query + -c + org.apache.druid.extensions:druid-catalog + -c + org.apache.druid.extensions:druid-protobuf-extensions + -c + org.apache.druid.extensions:postgresql-metadata-storage + -c + org.apache.druid.extensions:druid-s3-extensions + -c + org.apache.druid.extensions:druid-aws-rds-extensions + -c + org.apache.druid.extensions:simple-client-sslcontext + -c + org.apache.druid.extensions:druid-basic-security + -c + org.apache.druid.extensions:druid-pac4j + -c + org.apache.druid.extensions:druid-kubernetes-extensions + --clean + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + distro-assembly + package + + single + + + apache-druid-${project.parent.version} + posix + + src/assembly/assembly.xml + + + + + + + org.codehaus.mojo + license-maven-plugin + + + download-licenses + + download-licenses + + + + + + + + + + bundle-contrib-exts-used + + + + org.codehaus.mojo + exec-maven-plugin + + + pull-deps-contrib-exts + package + + exec + + + ${project.parent.basedir}/examples/bin/run-java + + -classpath + + -Ddruid.extensions.loadList=[] + -Ddruid.extensions.directory=${project.build.directory}/extensions + + + -Ddruid.extensions.hadoopDependenciesDir=${project.build.directory}/hadoop-dependencies + + org.apache.druid.cli.Main + tools + pull-deps + --defaultVersion + ${project.parent.version} + -l + ${settings.localRepository} + --no-default-hadoop + -c + org.apache.druid.extensions.contrib:kafka-emitter + -c + org.apache.druid.extensions.contrib:statsd-emitter + -c + org.apache.druid.extensions.contrib:prometheus-emitter + -c + org.apache.druid.extensions.contrib:opentelemetry-emitter + -c + org.apache.druid.extensions.contrib:druid-opencensus-extensions + -c + io.confluent.druid.extensions:confluent-extensions + -c + org.apache.druid.extensions.contrib:opentelemetry-emitter + + + + + + + + From 5fdd6b95cbaa5a42f314302e51afb2e50ca1aea4 Mon Sep 17 00:00:00 2001 From: pagrawal Date: Mon, 27 Nov 2023 17:31:03 +0530 Subject: [PATCH 096/114] update parent pom version for Confluent extensions --- extensions-contrib/confluent-extensions/pom.xml | 2 +- extensions-contrib/opencensus-extensions/pom.xml | 2 +- extensions-contrib/opentelemetry-extensions/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/extensions-contrib/confluent-extensions/pom.xml b/extensions-contrib/confluent-extensions/pom.xml index 90dbecd53a49..35046934886a 100644 --- a/extensions-contrib/confluent-extensions/pom.xml +++ b/extensions-contrib/confluent-extensions/pom.xml @@ -17,7 +17,7 @@ druid org.apache.druid - 25.0.0 + 28.0.0 ../../pom.xml diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml index 0552675a8a21..d96bb6d17dad 100644 --- a/extensions-contrib/opencensus-extensions/pom.xml +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -31,7 +31,7 @@ druid org.apache.druid - 25.0.0 + 28.0.0 ../../pom.xml diff --git a/extensions-contrib/opentelemetry-extensions/pom.xml b/extensions-contrib/opentelemetry-extensions/pom.xml index 971d2d9983f9..78171e0a2f89 100644 --- a/extensions-contrib/opentelemetry-extensions/pom.xml +++ b/extensions-contrib/opentelemetry-extensions/pom.xml @@ -30,7 +30,7 @@ druid org.apache.druid - 25.0.0 + 28.0.0 ../../pom.xml From 38bf7e264aa9e783dd2c568811ab792220f7410e Mon Sep 17 00:00:00 2001 From: pagrawal Date: Tue, 28 Nov 2023 13:09:42 +0530 Subject: [PATCH 097/114] Add value to child POMs --- extensions-contrib/opencensus-extensions/pom.xml | 1 + extensions-contrib/opentelemetry-extensions/pom.xml | 1 + .../common/task/batch/parallel/distribution/StringSketch.java | 2 +- .../segment/realtime/firehose/IngestSegmentFirehoseTest.java | 3 +-- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml index d96bb6d17dad..b4f3ac0398dd 100644 --- a/extensions-contrib/opencensus-extensions/pom.xml +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -64,6 +64,7 @@ io.opentelemetry.proto opentelemetry-proto + 0.19.0-alpha org.apache.druid.extensions.contrib diff --git a/extensions-contrib/opentelemetry-extensions/pom.xml b/extensions-contrib/opentelemetry-extensions/pom.xml index 78171e0a2f89..8008a2fa63af 100644 --- a/extensions-contrib/opentelemetry-extensions/pom.xml +++ b/extensions-contrib/opentelemetry-extensions/pom.xml @@ -41,6 +41,7 @@ io.opentelemetry.proto opentelemetry-proto + 0.19.0-alpha com.google.guava diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/distribution/StringSketch.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/distribution/StringSketch.java index e2794f6b1caf..b9400930a26f 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/distribution/StringSketch.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/distribution/StringSketch.java @@ -242,4 +242,4 @@ public StringSketch deserialize(JsonParser jsonParser, DeserializationContext de } } } -} +} \ No newline at end of file diff --git a/server/src/test/java/org/apache/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java b/server/src/test/java/org/apache/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java index 4a3d827b7c25..44f24f879a22 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/firehose/IngestSegmentFirehoseTest.java @@ -22,7 +22,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import org.apache.druid.collections.spatial.search.RadiusBound; -import org.apache.druid.common.config.NullHandlingTest; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.impl.DelimitedParseSpec; import org.apache.druid.data.input.impl.DimensionsSpec; @@ -67,7 +66,7 @@ /** */ @RunWith(Parameterized.class) -public class IngestSegmentFirehoseTest extends NullHandlingTest +public class IngestSegmentFirehoseTest { private static final DimensionsSpec DIMENSIONS_SPEC = new DimensionsSpec( ImmutableList.of( From dac8ac699292e81918c96f7b434f85c9485a645d Mon Sep 17 00:00:00 2001 From: mustajibmk <120099779+mustajibmk@users.noreply.github.com> Date: Tue, 28 Nov 2023 17:22:00 +0530 Subject: [PATCH 098/114] Upgrade dependencies to match upstream v28 & checkstyle fix --- .../druid/data/input/kafkainput/KafkaInputFormat.java | 3 +-- .../druid/data/input/kafkainput/KafkaInputReader.java | 2 +- .../druid/data/input/kafkainput/KafkaInputFormatTest.java | 1 + .../java/org/apache/druid/storage/s3/S3TaskLogsTest.java | 1 + .../task/batch/parallel/distribution/StringSketch.java | 2 +- pom.xml | 6 +++--- 6 files changed, 8 insertions(+), 7 deletions(-) diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputFormat.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputFormat.java index 129966705740..f8fbcbe153a7 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputFormat.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputFormat.java @@ -120,8 +120,7 @@ record -> temporaryDirectory ), keyColumnName, - timestampColumnName, - topicColumnName + timestampColumnName ); } diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputReader.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputReader.java index 9895f0f67432..458955e58070 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputReader.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputReader.java @@ -192,4 +192,4 @@ public CloseableIterator sample() throws IOException { return read().map(row -> InputRowListPlusRawValues.of(row, ((MapBasedInputRow) row).getEvent())); } -} \ No newline at end of file +} diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/data/input/kafkainput/KafkaInputFormatTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/data/input/kafkainput/KafkaInputFormatTest.java index 0321a4da4218..9f7afe482a35 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/data/input/kafkainput/KafkaInputFormatTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/data/input/kafkainput/KafkaInputFormatTest.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import org.apache.druid.data.input.ColumnsFilter; import org.apache.druid.data.input.InputEntityReader; diff --git a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3TaskLogsTest.java b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3TaskLogsTest.java index bd0067ee9dfe..bc1c7e66cf1c 100644 --- a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3TaskLogsTest.java +++ b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3TaskLogsTest.java @@ -29,6 +29,7 @@ import com.amazonaws.services.s3.model.Owner; import com.amazonaws.services.s3.model.Permission; import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.model.PutObjectResult; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.google.common.base.Optional; diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/distribution/StringSketch.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/distribution/StringSketch.java index b9400930a26f..e2794f6b1caf 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/distribution/StringSketch.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/distribution/StringSketch.java @@ -242,4 +242,4 @@ public StringSketch deserialize(JsonParser jsonParser, DeserializationContext de } } } -} \ No newline at end of file +} diff --git a/pom.xml b/pom.xml index acc10c8ebf85..b84fe178590c 100644 --- a/pom.xml +++ b/pom.xml @@ -80,14 +80,14 @@ 2.0.0 2.2.4 2.13.9 - 1.17.0 + 1.23.0 1.11.3 - 1.21.0 - 3.2.0 + 1.35.0 + 4.2.0 2.0.0 10.14.2.0 4.2.19 From ee74d314c9f7fbf201fc543f5bb24246d6eb6557 Mon Sep 17 00:00:00 2001 From: pagrawal Date: Mon, 4 Dec 2023 11:35:17 +0530 Subject: [PATCH 099/114] KafkaEmitter changes --- .../druid/emitter/kafka/KafkaEmitter.java | 14 +- .../emitter/kafka/KafkaEmitterConfigTest.java | 2 +- .../druid/emitter/kafka/KafkaEmitterTest.java | 2 +- pom.xml | 22 +- .../refresh-button/refresh-button.tsx | 4 +- .../views/ingestion-view/ingestion-view.tsx | 1263 ----------------- 6 files changed, 28 insertions(+), 1279 deletions(-) delete mode 100644 web-console/src/views/ingestion-view/ingestion-view.tsx diff --git a/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitter.java b/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitter.java index 54f563eb3921..7a9a70e5f887 100644 --- a/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitter.java +++ b/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitter.java @@ -66,10 +66,10 @@ public class KafkaEmitter implements Emitter private final KafkaEmitterConfig config; private final Producer producer; private final ObjectMapper jsonMapper; - private final MemoryBoundLinkedBlockingQueue metricQueue; - private final MemoryBoundLinkedBlockingQueue alertQueue; - private final MemoryBoundLinkedBlockingQueue requestQueue; - private final MemoryBoundLinkedBlockingQueue segmentMetadataQueue; + private final MemoryBoundLinkedBlockingQueue metricQueue; + private final MemoryBoundLinkedBlockingQueue alertQueue; + private final MemoryBoundLinkedBlockingQueue requestQueue; + private final MemoryBoundLinkedBlockingQueue segmentMetadataQueue; private final ScheduledExecutorService scheduler; protected int sendInterval = DEFAULT_SEND_INTERVAL_SECONDS; @@ -173,13 +173,13 @@ private void sendSegmentMetadataToKafka() sendToKafka(config.getSegmentMetadataTopic(), segmentMetadataQueue, setProducerCallback(segmentMetadataLost)); } - private void sendToKafka(final String topic, MemoryBoundLinkedBlockingQueue recordQueue, Callback callback) + private void sendToKafka(final String topic, MemoryBoundLinkedBlockingQueue recordQueue, Callback callback) { - ObjectContainer objectToSend; + ObjectContainer objectToSend; try { while (true) { objectToSend = recordQueue.take(); - producer.send(new ProducerRecord<>(topic, objectToSend.getData()), callback); + producer.send(new ProducerRecord<>(topic, objectToSend.getData().toString()), callback); } } catch (Throwable e) { diff --git a/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterConfigTest.java b/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterConfigTest.java index dc02349cb181..3a39d461f19d 100644 --- a/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterConfigTest.java +++ b/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterConfigTest.java @@ -60,7 +60,7 @@ public void testSerDeserKafkaEmitterConfig() throws IOException public void testSerDeserKafkaEmitterConfigNullRequestTopic() throws IOException { KafkaEmitterConfig kafkaEmitterConfig = new KafkaEmitterConfig("hostname", null, "metricTest", - "alertTest", null, "metadataTest",null, + "alertTest", null, "metadataTest", null, "clusterNameTest", ImmutableMap.builder() .put("testKey", "testValue").build() ); diff --git a/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterTest.java b/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterTest.java index b5196018d784..051f0a649af5 100644 --- a/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterTest.java +++ b/extensions-contrib/kafka-emitter/src/test/java/org/apache/druid/emitter/kafka/KafkaEmitterTest.java @@ -106,7 +106,7 @@ public void testKafkaEmitter() throws InterruptedException ObjectMapper mapper = new ObjectMapper(); mapper.registerModule(new JodaModule()); final KafkaEmitter kafkaEmitter = new KafkaEmitter( - new KafkaEmitterConfig("", eventsType, "metrics", "alerts", requestTopic, "metadata", null,"test-cluster", null), + new KafkaEmitterConfig("", eventsType, "metrics", "alerts", requestTopic, "metadata", null, "test-cluster", null), mapper ) { diff --git a/pom.xml b/pom.xml index b84fe178590c..50b54254e130 100644 --- a/pom.xml +++ b/pom.xml @@ -66,7 +66,7 @@ scm:git:ssh://git@github.com/apache/druid.git scm:git:ssh://git@github.com/apache/druid.git https://github.com/apache/druid.git - 0.19.0-SNAPSHOT + druid-28.0.0 @@ -131,9 +131,9 @@ v1-rev20230301-2.0.0 - maven.org - Maven Central Repository - https://repo1.maven.org/maven2/ + apache.snapshots + Apache Snapshot Repository + https://repository.apache.org/snapshots 0.19.0-alpha @@ -2169,5 +2169,19 @@ true + + hadoop3 + + + hadoop3.enabled + true + + + + 3.3.1 + 5.3.6.Final + 4.5.13 + + diff --git a/web-console/src/components/refresh-button/refresh-button.tsx b/web-console/src/components/refresh-button/refresh-button.tsx index 5a5c65151d3e..92ad3fb9bdee 100644 --- a/web-console/src/components/refresh-button/refresh-button.tsx +++ b/web-console/src/components/refresh-button/refresh-button.tsx @@ -39,12 +39,10 @@ export interface RefreshButtonProps { } export const RefreshButton = React.memo(function RefreshButton(props: RefreshButtonProps) { - const { onRefresh, localStorageKey, defaultDelay = 30000 } = props; - return ( ; - - resumeSupervisorId?: string; - suspendSupervisorId?: string; - resetSupervisorId?: string; - terminateSupervisorId?: string; - - showResumeAllSupervisors: boolean; - showSuspendAllSupervisors: boolean; - showTerminateAllSupervisors: boolean; - - tasksState: QueryState; - - taskFilter: Filter[]; - supervisorFilter: Filter[]; - - groupTasksBy?: 'group_id' | 'type' | 'datasource' | 'status'; - - killTaskId?: string; - - supervisorSpecDialogOpen: boolean; - taskSpecDialogOpen: boolean; - alertErrorMsg?: string; - - taskTableActionDialogId?: string; - taskTableActionDialogStatus?: string; - taskTableActionDialogActions: BasicAction[]; - supervisorTableActionDialogId?: string; - supervisorTableActionDialogActions: BasicAction[]; - hiddenTaskColumns: LocalStorageBackedVisibility; - hiddenSupervisorColumns: LocalStorageBackedVisibility; -} - -function statusToColor(status: string): string { - switch (status) { - case 'RUNNING': - return '#2167d5'; - case 'WAITING': - return '#d5631a'; - case 'PENDING': - return '#ffbf00'; - case 'SUCCESS': - return '#57d500'; - case 'FAILED': - return '#d5100a'; - case 'CANCELED': - return '#858585'; - default: - return '#0a1500'; - } -} - -function stateToColor(status: string): string { - switch (status) { - case 'UNHEALTHY_SUPERVISOR': - return '#d5100a'; - case 'UNHEALTHY_TASKS': - return '#d5100a'; - case 'PENDING': - return '#ffbf00'; - case `SUSPENDED`: - return '#ffbf00'; - case 'STOPPING': - return '#d5100a'; - case 'RUNNING': - return '#2167d5'; - default: - return '#0a1500'; - } -} - -export class IngestionView extends React.PureComponent { - private readonly supervisorQueryManager: QueryManager; - private readonly taskQueryManager: QueryManager; - static statusRanking: Record = { - RUNNING: 4, - PENDING: 3, - WAITING: 2, - SUCCESS: 1, - FAILED: 1, - }; - - static SUPERVISOR_SQL = `SELECT - "supervisor_id", "type", "source", "state", "detailed_state", "suspended" = 1 AS "suspended" -FROM sys.supervisors -ORDER BY "supervisor_id"`; - - static TASK_SQL = `WITH tasks AS (SELECT - "task_id", "group_id", "type", "datasource", "created_time", "location", "duration", "error_msg", - CASE WHEN "error_msg" = '${CANCELED_ERROR_MSG}' THEN 'CANCELED' WHEN "status" = 'RUNNING' THEN "runner_status" ELSE "status" END AS "status" - FROM sys.tasks -) -SELECT "task_id", "group_id", "type", "datasource", "created_time", "location", "duration", "error_msg", "status" -FROM tasks -ORDER BY - ( - CASE "status" - WHEN 'RUNNING' THEN 4 - WHEN 'PENDING' THEN 3 - WHEN 'WAITING' THEN 2 - ELSE 1 - END - ) DESC, - "created_time" DESC`; - - constructor(props: IngestionViewProps, context: any) { - super(props, context); - - const taskFilter: Filter[] = []; - if (props.taskId) taskFilter.push({ id: 'task_id', value: `=${props.taskId}` }); - if (props.taskGroupId) taskFilter.push({ id: 'group_id', value: `=${props.taskGroupId}` }); - if (props.datasourceId) taskFilter.push({ id: 'datasource', value: `=${props.datasourceId}` }); - - const supervisorFilter: Filter[] = []; - if (props.datasourceId) - supervisorFilter.push({ id: 'datasource', value: `=${props.datasourceId}` }); - - this.state = { - supervisorsState: QueryState.INIT, - - showResumeAllSupervisors: false, - showSuspendAllSupervisors: false, - showTerminateAllSupervisors: false, - - tasksState: QueryState.INIT, - taskFilter: taskFilter, - supervisorFilter: supervisorFilter, - - supervisorSpecDialogOpen: props.openDialog === 'supervisor', - taskSpecDialogOpen: props.openDialog === 'task', - - taskTableActionDialogActions: [], - supervisorTableActionDialogActions: [], - - hiddenTaskColumns: new LocalStorageBackedVisibility( - LocalStorageKeys.TASK_TABLE_COLUMN_SELECTION, - ), - hiddenSupervisorColumns: new LocalStorageBackedVisibility( - LocalStorageKeys.SUPERVISOR_TABLE_COLUMN_SELECTION, - ), - }; - - this.supervisorQueryManager = new QueryManager({ - processQuery: async capabilities => { - if (capabilities.hasSql()) { - return await queryDruidSql({ - query: IngestionView.SUPERVISOR_SQL, - }); - } else if (capabilities.hasOverlordAccess()) { - const supervisors = (await Api.instance.get('/druid/indexer/v1/supervisor?full')).data; - if (!Array.isArray(supervisors)) throw new Error(`Unexpected results`); - return supervisors.map((sup: any) => { - return { - supervisor_id: deepGet(sup, 'id'), - type: deepGet(sup, 'spec.tuningConfig.type'), - source: - deepGet(sup, 'spec.ioConfig.topic') || - deepGet(sup, 'spec.ioConfig.stream') || - 'n/a', - state: deepGet(sup, 'state'), - detailed_state: deepGet(sup, 'detailedState'), - suspended: Boolean(deepGet(sup, 'suspended')), - }; - }); - } else { - throw new Error(`must have SQL or overlord access`); - } - }, - onStateChange: supervisorsState => { - this.setState({ - supervisorsState, - }); - }, - }); - - this.taskQueryManager = new QueryManager({ - processQuery: async capabilities => { - if (capabilities.hasSql()) { - return await queryDruidSql({ - query: IngestionView.TASK_SQL, - }); - } else if (capabilities.hasOverlordAccess()) { - const resp = await Api.instance.get(`/druid/indexer/v1/tasks`); - return IngestionView.parseTasks(resp.data); - } else { - throw new Error(`must have SQL or overlord access`); - } - }, - onStateChange: tasksState => { - this.setState({ - tasksState, - }); - }, - }); - } - - static parseTasks = (data: any[]): TaskQueryResultRow[] => { - return data.map(d => { - return { - task_id: d.id, - group_id: d.groupId, - type: d.type, - created_time: d.createdTime, - datasource: d.dataSource, - duration: d.duration ? d.duration : 0, - error_msg: d.errorMsg, - location: d.location.host ? `${d.location.host}:${d.location.port}` : null, - status: d.statusCode === 'RUNNING' ? d.runnerStatusCode : d.statusCode, - }; - }); - }; - - private static onSecondaryPaneSizeChange(secondaryPaneSize: number) { - localStorageSet(LocalStorageKeys.INGESTION_VIEW_PANE_SIZE, String(secondaryPaneSize)); - } - - componentDidMount(): void { - const { capabilities } = this.props; - - this.supervisorQueryManager.runQuery(capabilities); - this.taskQueryManager.runQuery(capabilities); - } - - componentWillUnmount(): void { - this.supervisorQueryManager.terminate(); - this.taskQueryManager.terminate(); - } - - private readonly closeSpecDialogs = () => { - this.setState({ - supervisorSpecDialogOpen: false, - taskSpecDialogOpen: false, - }); - }; - - private readonly submitSupervisor = async (spec: JSON) => { - try { - await Api.instance.post('/druid/indexer/v1/supervisor', spec); - } catch (e) { - AppToaster.show({ - message: `Failed to submit supervisor: ${getDruidErrorMessage(e)}`, - intent: Intent.DANGER, - }); - return; - } - - AppToaster.show({ - message: 'Supervisor submitted successfully', - intent: Intent.SUCCESS, - }); - this.supervisorQueryManager.rerunLastQuery(); - }; - - private readonly submitTask = async (spec: JSON) => { - try { - await Api.instance.post('/druid/indexer/v1/task', spec); - } catch (e) { - AppToaster.show({ - message: `Failed to submit task: ${getDruidErrorMessage(e)}`, - intent: Intent.DANGER, - }); - return; - } - - AppToaster.show({ - message: 'Task submitted successfully', - intent: Intent.SUCCESS, - }); - this.taskQueryManager.rerunLastQuery(); - }; - - private getSupervisorActions( - id: string, - supervisorSuspended: boolean, - type: string, - ): BasicAction[] { - const { goToDatasource, goToStreamingDataLoader } = this.props; - - const actions: BasicAction[] = []; - if (oneOf(type, 'kafka', 'kinesis')) { - actions.push( - { - icon: IconNames.MULTI_SELECT, - title: 'Go to datasource', - onAction: () => goToDatasource(id), - }, - { - icon: IconNames.CLOUD_UPLOAD, - title: 'Open in data loader', - onAction: () => goToStreamingDataLoader(id), - }, - ); - } - actions.push( - { - icon: supervisorSuspended ? IconNames.PLAY : IconNames.PAUSE, - title: supervisorSuspended ? 'Resume' : 'Suspend', - onAction: () => - supervisorSuspended - ? this.setState({ resumeSupervisorId: id }) - : this.setState({ suspendSupervisorId: id }), - }, - { - icon: IconNames.STEP_BACKWARD, - title: 'Hard reset', - intent: Intent.DANGER, - onAction: () => this.setState({ resetSupervisorId: id }), - }, - { - icon: IconNames.CROSS, - title: 'Terminate', - intent: Intent.DANGER, - onAction: () => this.setState({ terminateSupervisorId: id }), - }, - ); - return actions; - } - - renderResumeSupervisorAction() { - const { resumeSupervisorId } = this.state; - if (!resumeSupervisorId) return; - - return ( - { - const resp = await Api.instance.post( - `/druid/indexer/v1/supervisor/${Api.encodePath(resumeSupervisorId)}/resume`, - {}, - ); - return resp.data; - }} - confirmButtonText="Resume supervisor" - successText="Supervisor has been resumed" - failText="Could not resume supervisor" - intent={Intent.PRIMARY} - onClose={() => { - this.setState({ resumeSupervisorId: undefined }); - }} - onSuccess={() => { - this.supervisorQueryManager.rerunLastQuery(); - }} - > -

{`Are you sure you want to resume supervisor '${resumeSupervisorId}'?`}

- - ); - } - - renderSuspendSupervisorAction() { - const { suspendSupervisorId } = this.state; - if (!suspendSupervisorId) return; - - return ( - { - const resp = await Api.instance.post( - `/druid/indexer/v1/supervisor/${Api.encodePath(suspendSupervisorId)}/suspend`, - {}, - ); - return resp.data; - }} - confirmButtonText="Suspend supervisor" - successText="Supervisor has been suspended" - failText="Could not suspend supervisor" - intent={Intent.DANGER} - onClose={() => { - this.setState({ suspendSupervisorId: undefined }); - }} - onSuccess={() => { - this.supervisorQueryManager.rerunLastQuery(); - }} - > -

{`Are you sure you want to suspend supervisor '${suspendSupervisorId}'?`}

- - ); - } - - renderResetSupervisorAction() { - const { resetSupervisorId } = this.state; - if (!resetSupervisorId) return; - - return ( - { - const resp = await Api.instance.post( - `/druid/indexer/v1/supervisor/${Api.encodePath(resetSupervisorId)}/reset`, - {}, - ); - return resp.data; - }} - confirmButtonText="Hard reset supervisor" - successText="Supervisor has been hard reset" - failText="Could not hard reset supervisor" - intent={Intent.DANGER} - onClose={() => { - this.setState({ resetSupervisorId: undefined }); - }} - onSuccess={() => { - this.supervisorQueryManager.rerunLastQuery(); - }} - warningChecks={[ - `I understand that resetting ${resetSupervisorId} will clear checkpoints and therefore lead to data loss or duplication.`, - 'I understand that this operation cannot be undone.', - ]} - > -

{`Are you sure you want to hard reset supervisor '${resetSupervisorId}'?`}

-

Hard resetting a supervisor will lead to data loss or data duplication.

-

- The reason for using this operation is to recover from a state in which the supervisor - ceases operating due to missing offsets. -

- - ); - } - - renderTerminateSupervisorAction() { - const { terminateSupervisorId } = this.state; - if (!terminateSupervisorId) return; - - return ( - { - const resp = await Api.instance.post( - `/druid/indexer/v1/supervisor/${Api.encodePath(terminateSupervisorId)}/terminate`, - {}, - ); - return resp.data; - }} - confirmButtonText="Terminate supervisor" - successText="Supervisor has been terminated" - failText="Could not terminate supervisor" - intent={Intent.DANGER} - onClose={() => { - this.setState({ terminateSupervisorId: undefined }); - }} - onSuccess={() => { - this.supervisorQueryManager.rerunLastQuery(); - }} - > -

{`Are you sure you want to terminate supervisor '${terminateSupervisorId}'?`}

-

This action is not reversible.

- - ); - } - - private renderSupervisorFilterableCell(field: string) { - const { supervisorFilter } = this.state; - - return (row: { value: any }) => ( - this.setState({ supervisorFilter: filters })} - > - {row.value} - - ); - } - - private onSupervisorDetail(supervisor: SupervisorQueryResultRow) { - this.setState({ - supervisorTableActionDialogId: supervisor.supervisor_id, - supervisorTableActionDialogActions: this.getSupervisorActions( - supervisor.supervisor_id, - supervisor.suspended, - supervisor.type, - ), - }); - } - - private renderSupervisorTable() { - const { supervisorsState, hiddenSupervisorColumns, taskFilter, supervisorFilter } = this.state; - - const supervisors = supervisorsState.data || []; - return ( - { - this.setState({ - supervisorFilter: filtered, - taskFilter: - column.id === 'datasource' - ? syncFilterClauseById(taskFilter, filtered, 'datasource') - : taskFilter, - }); - }} - filterable - defaultPageSize={SMALL_TABLE_PAGE_SIZE} - pageSizeOptions={SMALL_TABLE_PAGE_SIZE_OPTIONS} - showPagination={supervisors.length > SMALL_TABLE_PAGE_SIZE} - columns={[ - { - Header: 'Datasource', - id: 'datasource', - accessor: 'supervisor_id', - width: 300, - show: hiddenSupervisorColumns.shown('Datasource'), - Cell: ({ value, original }) => ( - this.onSupervisorDetail(original)} - hoverIcon={IconNames.EDIT} - > - {value} - - ), - }, - { - Header: 'Type', - accessor: 'type', - width: 100, - Cell: this.renderSupervisorFilterableCell('type'), - show: hiddenSupervisorColumns.shown('Type'), - }, - { - Header: 'Topic/Stream', - accessor: 'source', - width: 300, - Cell: this.renderSupervisorFilterableCell('source'), - show: hiddenSupervisorColumns.shown('Topic/Stream'), - }, - { - Header: 'Status', - id: 'status', - width: 300, - accessor: 'detailed_state', - Cell: row => ( - this.setState({ supervisorFilter: filters })} - > - - ●  - {row.value} - - - ), - show: hiddenSupervisorColumns.shown('Status'), - }, - { - Header: ACTION_COLUMN_LABEL, - id: ACTION_COLUMN_ID, - accessor: 'supervisor_id', - width: ACTION_COLUMN_WIDTH, - filterable: false, - Cell: row => { - const id = row.value; - const type = row.original.type; - const supervisorSuspended = row.original.suspended; - const supervisorActions = this.getSupervisorActions(id, supervisorSuspended, type); - return ( - this.onSupervisorDetail(row.original)} - actions={supervisorActions} - /> - ); - }, - show: hiddenSupervisorColumns.shown(ACTION_COLUMN_LABEL), - }, - ]} - /> - ); - } - - private getTaskActions( - id: string, - datasource: string, - status: string, - type: string, - ): BasicAction[] { - const { goToDatasource, goToClassicBatchDataLoader } = this.props; - - const actions: BasicAction[] = []; - if (datasource && status === 'SUCCESS') { - actions.push({ - icon: IconNames.MULTI_SELECT, - title: 'Go to datasource', - onAction: () => goToDatasource(datasource), - }); - } - if (oneOf(type, 'index', 'index_parallel')) { - actions.push({ - icon: IconNames.CLOUD_UPLOAD, - title: 'Open in data loader', - onAction: () => goToClassicBatchDataLoader(id), - }); - } - if (oneOf(status, 'RUNNING', 'WAITING', 'PENDING')) { - actions.push({ - icon: IconNames.CROSS, - title: 'Kill', - intent: Intent.DANGER, - onAction: () => this.setState({ killTaskId: id }), - }); - } - return actions; - } - - renderKillTaskAction() { - const { killTaskId } = this.state; - if (!killTaskId) return; - - return ( - { - const resp = await Api.instance.post( - `/druid/indexer/v1/task/${Api.encodePath(killTaskId)}/shutdown`, - {}, - ); - return resp.data; - }} - confirmButtonText="Kill task" - successText="Task was killed" - failText="Could not kill task" - intent={Intent.DANGER} - onClose={() => { - this.setState({ killTaskId: undefined }); - }} - onSuccess={() => { - this.taskQueryManager.rerunLastQuery(); - }} - > -

{`Are you sure you want to kill task '${killTaskId}'?`}

- - ); - } - - private renderTaskFilterableCell(field: string) { - const { taskFilter } = this.state; - - return (row: { value: any }) => ( - this.setState({ taskFilter: filters })} - > - {row.value} - - ); - } - - private onTaskDetail(task: TaskQueryResultRow) { - this.setState({ - taskTableActionDialogId: task.task_id, - taskTableActionDialogStatus: task.status, - taskTableActionDialogActions: this.getTaskActions( - task.task_id, - task.datasource, - task.status, - task.type, - ), - }); - } - - private renderTaskTable() { - const { tasksState, taskFilter, groupTasksBy, hiddenTaskColumns, supervisorFilter } = - this.state; - - const tasks = tasksState.data || []; - return ( - { - this.setState({ - supervisorFilter: - column.id === 'datasource' - ? syncFilterClauseById(supervisorFilter, filtered, 'datasource') - : supervisorFilter, - taskFilter: filtered, - }); - }} - defaultSorted={[{ id: 'status', desc: true }]} - pivotBy={groupTasksBy ? [groupTasksBy] : []} - defaultPageSize={SMALL_TABLE_PAGE_SIZE} - pageSizeOptions={SMALL_TABLE_PAGE_SIZE_OPTIONS} - showPagination={tasks.length > SMALL_TABLE_PAGE_SIZE} - columns={[ - { - Header: 'Task ID', - accessor: 'task_id', - width: 440, - Cell: ({ value, original }) => ( - this.onTaskDetail(original)} - hoverIcon={IconNames.EDIT} - > - {value} - - ), - Aggregated: () => '', - show: hiddenTaskColumns.shown('Task ID'), - }, - { - Header: 'Group ID', - accessor: 'group_id', - width: 300, - Cell: this.renderTaskFilterableCell('group_id'), - Aggregated: () => '', - show: hiddenTaskColumns.shown('Group ID'), - }, - { - Header: 'Type', - accessor: 'type', - width: 140, - Cell: this.renderTaskFilterableCell('type'), - show: hiddenTaskColumns.shown('Type'), - }, - { - Header: 'Datasource', - accessor: 'datasource', - width: 200, - Cell: this.renderTaskFilterableCell('datasource'), - show: hiddenTaskColumns.shown('Datasource'), - }, - { - Header: 'Status', - id: 'status', - width: 110, - accessor: row => ({ - status: row.status, - created_time: row.created_time, - toString: () => row.status, - }), - Cell: row => { - if (row.aggregated) return ''; - const { status } = row.original; - const errorMsg = row.original.error_msg; - return ( - this.setState({ taskFilter: filters })} - > - - ●  - {status} - {errorMsg && errorMsg !== CANCELED_ERROR_MSG && ( - this.setState({ alertErrorMsg: errorMsg })} - title={errorMsg} - > -  ? - - )} - - - ); - }, - sortMethod: (d1, d2) => { - const typeofD1 = typeof d1; - const typeofD2 = typeof d2; - if (typeofD1 !== typeofD2) return 0; - switch (typeofD1) { - case 'string': - return IngestionView.statusRanking[d1] - IngestionView.statusRanking[d2]; - - case 'object': - return ( - IngestionView.statusRanking[d1.status] - - IngestionView.statusRanking[d2.status] || - d1.created_time.localeCompare(d2.created_time) - ); - - default: - return 0; - } - }, - show: hiddenTaskColumns.shown('Status'), - }, - { - Header: 'Created time', - accessor: 'created_time', - width: 190, - Cell: this.renderTaskFilterableCell('created_time'), - Aggregated: () => '', - show: hiddenTaskColumns.shown('Created time'), - }, - { - Header: 'Duration', - accessor: 'duration', - width: 80, - filterable: false, - className: 'padded', - Cell({ value, original, aggregated }) { - if (aggregated) return ''; - if (value > 0) { - return formatDuration(value); - } - if (oneOf(original.status, 'RUNNING', 'PENDING') && original.created_time) { - // Compute running duration from the created time if it exists - return formatDuration(Date.now() - Date.parse(original.created_time)); - } - return ''; - }, - Aggregated: () => '', - show: hiddenTaskColumns.shown('Duration'), - }, - { - Header: 'Location', - accessor: 'location', - width: 200, - Cell: this.renderTaskFilterableCell('location'), - Aggregated: () => '', - show: hiddenTaskColumns.shown('Location'), - }, - { - Header: ACTION_COLUMN_LABEL, - id: ACTION_COLUMN_ID, - accessor: 'task_id', - width: ACTION_COLUMN_WIDTH, - filterable: false, - Cell: row => { - if (row.aggregated) return ''; - const id = row.value; - const type = row.row.type; - const { datasource, status } = row.original; - const taskActions = this.getTaskActions(id, datasource, status, type); - return ( - this.onTaskDetail(row.original)} - actions={taskActions} - /> - ); - }, - Aggregated: () => '', - show: hiddenTaskColumns.shown(ACTION_COLUMN_LABEL), - }, - ]} - /> - ); - } - - renderBulkSupervisorActions() { - const { capabilities, goToQuery } = this.props; - - return ( - <> - - {capabilities.hasSql() && ( - goToQuery({ queryString: IngestionView.SUPERVISOR_SQL })} - /> - )} - this.setState({ supervisorSpecDialogOpen: true })} - /> - this.setState({ showResumeAllSupervisors: true })} - /> - this.setState({ showSuspendAllSupervisors: true })} - /> - this.setState({ showTerminateAllSupervisors: true })} - /> - - {this.renderResumeAllSupervisorAction()} - {this.renderSuspendAllSupervisorAction()} - {this.renderTerminateAllSupervisorAction()} - - ); - } - - renderResumeAllSupervisorAction() { - const { showResumeAllSupervisors } = this.state; - if (!showResumeAllSupervisors) return; - - return ( - { - const resp = await Api.instance.post(`/druid/indexer/v1/supervisor/resumeAll`, {}); - return resp.data; - }} - confirmButtonText="Resume all supervisors" - successText="All supervisors have been resumed" - failText="Could not resume all supervisors" - intent={Intent.PRIMARY} - onClose={() => { - this.setState({ showResumeAllSupervisors: false }); - }} - onSuccess={() => { - this.supervisorQueryManager.rerunLastQuery(); - }} - > -

Are you sure you want to resume all the supervisors?

- - ); - } - - renderSuspendAllSupervisorAction() { - const { showSuspendAllSupervisors } = this.state; - if (!showSuspendAllSupervisors) return; - - return ( - { - const resp = await Api.instance.post(`/druid/indexer/v1/supervisor/suspendAll`, {}); - return resp.data; - }} - confirmButtonText="Suspend all supervisors" - successText="All supervisors have been suspended" - failText="Could not suspend all supervisors" - intent={Intent.DANGER} - onClose={() => { - this.setState({ showSuspendAllSupervisors: false }); - }} - onSuccess={() => { - this.supervisorQueryManager.rerunLastQuery(); - }} - > -

Are you sure you want to suspend all the supervisors?

- - ); - } - - renderTerminateAllSupervisorAction() { - const { showTerminateAllSupervisors } = this.state; - if (!showTerminateAllSupervisors) return; - - return ( - { - const resp = await Api.instance.post(`/druid/indexer/v1/supervisor/terminateAll`, {}); - return resp.data; - }} - confirmButtonText="Terminate all supervisors" - successText="All supervisors have been terminated" - failText="Could not terminate all supervisors" - intent={Intent.DANGER} - onClose={() => { - this.setState({ showTerminateAllSupervisors: false }); - }} - onSuccess={() => { - this.supervisorQueryManager.rerunLastQuery(); - }} - > -

Are you sure you want to terminate all the supervisors?

- - ); - } - - renderBulkTasksActions() { - const { goToQuery, capabilities } = this.props; - - return ( - - {capabilities.hasSql() && ( - goToQuery({ queryString: IngestionView.TASK_SQL })} - /> - )} - this.setState({ taskSpecDialogOpen: true })} - /> - - ); - } - - render(): JSX.Element { - const { - groupTasksBy, - supervisorSpecDialogOpen, - taskSpecDialogOpen, - alertErrorMsg, - taskTableActionDialogId, - taskTableActionDialogActions, - supervisorTableActionDialogId, - supervisorTableActionDialogActions, - taskTableActionDialogStatus, - hiddenSupervisorColumns, - hiddenTaskColumns, - } = this.state; - - return ( - <> - -
- - { - if (auto && hasPopoverOpen()) return; - this.supervisorQueryManager.rerunLastQuery(auto); - }} - /> - {this.renderBulkSupervisorActions()} - - this.setState(prevState => ({ - hiddenSupervisorColumns: prevState.hiddenSupervisorColumns.toggle(column), - })) - } - tableColumnsHidden={hiddenSupervisorColumns.getHiddenColumns()} - /> - - {this.renderSupervisorTable()} -
-
- - - - - - - - - - { - if (auto && hasPopoverOpen()) return; - this.taskQueryManager.rerunLastQuery(auto); - }} - /> - {this.renderBulkTasksActions()} - - this.setState(prevState => ({ - hiddenTaskColumns: prevState.hiddenTaskColumns.toggle(column), - })) - } - tableColumnsHidden={hiddenTaskColumns.getHiddenColumns()} - /> - - {this.renderTaskTable()} -
- - {this.renderResumeSupervisorAction()} - {this.renderSuspendSupervisorAction()} - {this.renderResetSupervisorAction()} - {this.renderTerminateSupervisorAction()} - {this.renderKillTaskAction()} - {supervisorSpecDialogOpen && ( - - )} - {taskSpecDialogOpen && ( - - )} - this.setState({ alertErrorMsg: undefined })} - > -

{alertErrorMsg}

- - {supervisorTableActionDialogId && ( - this.setState({ supervisorTableActionDialogId: undefined })} - /> - )} - {taskTableActionDialogId && taskTableActionDialogStatus && ( - this.setState({ taskTableActionDialogId: undefined })} - /> - )} - - ); - } -} From 6103506a9429a6d387260a76108df1d1d4457d01 Mon Sep 17 00:00:00 2001 From: pagrawal Date: Mon, 4 Dec 2023 11:38:50 +0530 Subject: [PATCH 100/114] Modifying RowFunction interface --- .../confluent-extensions/pom.xml | 14 ++++++------ .../ExtractTenantTopicTransform.java | 22 ++++++++++++------- .../opencensus-extensions/pom.xml | 4 ++-- .../opentelemetry-extensions/pom.xml | 4 ++-- .../druid/segment/transform/RowFunction.java | 7 ++++-- 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/extensions-contrib/confluent-extensions/pom.xml b/extensions-contrib/confluent-extensions/pom.xml index 35046934886a..ed012b79e629 100644 --- a/extensions-contrib/confluent-extensions/pom.xml +++ b/extensions-contrib/confluent-extensions/pom.xml @@ -17,17 +17,11 @@ druid org.apache.druid - 28.0.0 + 28.0.0-SNAPSHOT ../../pom.xml - - org.apache.druid - druid-core - ${project.parent.version} - provided - org.apache.druid druid-processing @@ -72,5 +66,11 @@ test test-jar + + org.apache.druid + druid-processing + 28.0.0-SNAPSHOT + compile + diff --git a/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTopicTransform.java b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTopicTransform.java index 914d1cebc3cb..ef4b78b0f753 100644 --- a/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTopicTransform.java +++ b/extensions-contrib/confluent-extensions/src/main/java/io/confluent/druid/transform/ExtractTenantTopicTransform.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; +import org.apache.druid.data.input.Row; import org.apache.druid.segment.transform.RowFunction; import org.apache.druid.segment.transform.Transform; @@ -43,15 +44,20 @@ public String getFieldName() @Override public RowFunction getRowFunction() { - return row -> { - Object existing = row.getRaw(name); - // do not overwrite existing values if present - if (existing != null) { - return existing; - } + return new RowFunction() + { + @Override + public Object eval(Row row) + { + Object existing = row.getRaw(name); + // do not overwrite existing values if present + if (existing != null) { + return existing; + } - Object value = row.getRaw(fieldName); - return value == null ? null : TenantUtils.extractTenantTopic(value.toString()); + Object value = row.getRaw(fieldName); + return value == null ? null : TenantUtils.extractTenantTopic(value.toString()); + } }; } diff --git a/extensions-contrib/opencensus-extensions/pom.xml b/extensions-contrib/opencensus-extensions/pom.xml index b4f3ac0398dd..6e23a49a5786 100644 --- a/extensions-contrib/opencensus-extensions/pom.xml +++ b/extensions-contrib/opencensus-extensions/pom.xml @@ -31,7 +31,7 @@ druid org.apache.druid - 28.0.0 + 28.0.0-SNAPSHOT ../../pom.xml @@ -51,7 +51,7 @@ org.apache.druid - druid-core + druid-processing ${project.parent.version} provided diff --git a/extensions-contrib/opentelemetry-extensions/pom.xml b/extensions-contrib/opentelemetry-extensions/pom.xml index 8008a2fa63af..1a457cbc538a 100644 --- a/extensions-contrib/opentelemetry-extensions/pom.xml +++ b/extensions-contrib/opentelemetry-extensions/pom.xml @@ -30,7 +30,7 @@ druid org.apache.druid - 28.0.0 + 28.0.0-SNAPSHOT ../../pom.xml @@ -70,7 +70,7 @@ org.apache.druid - druid-core + druid-processing ${project.parent.version} provided diff --git a/processing/src/main/java/org/apache/druid/segment/transform/RowFunction.java b/processing/src/main/java/org/apache/druid/segment/transform/RowFunction.java index 375aa5f1b10f..4aab82781aad 100644 --- a/processing/src/main/java/org/apache/druid/segment/transform/RowFunction.java +++ b/processing/src/main/java/org/apache/druid/segment/transform/RowFunction.java @@ -20,6 +20,7 @@ package org.apache.druid.segment.transform; import org.apache.druid.data.input.Row; +import org.apache.druid.data.input.Rows; import java.util.List; @@ -29,6 +30,8 @@ public interface RowFunction { Object eval(Row row); - - List evalDimension(Row row); + default List evalDimension(Row row) + { + return Rows.objectToStrings(eval(row)); + } } From b939580488bedeea844971d0d94a579ab6c5052a Mon Sep 17 00:00:00 2001 From: Pankaj kumar Date: Mon, 4 Dec 2023 13:14:23 +0530 Subject: [PATCH 101/114] Fix test cases --- .../input/kafkainput/KafkaInputFormat.java | 3 +- .../input/kafkainput/KafkaInputReader.java | 294 ++++++++++++++---- .../kafkainput/KafkaInputFormatTest.java | 6 +- 3 files changed, 235 insertions(+), 68 deletions(-) diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputFormat.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputFormat.java index f8fbcbe153a7..129966705740 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputFormat.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputFormat.java @@ -120,7 +120,8 @@ record -> temporaryDirectory ), keyColumnName, - timestampColumnName + timestampColumnName, + topicColumnName ); } diff --git a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputReader.java b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputReader.java index 458955e58070..31b7cf66be19 100644 --- a/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputReader.java +++ b/extensions-core/kafka-indexing-service/src/main/java/org/apache/druid/data/input/kafkainput/KafkaInputReader.java @@ -20,34 +20,35 @@ package org.apache.druid.data.input.kafkainput; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import org.apache.druid.data.input.InputEntityReader; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRowListPlusRawValues; import org.apache.druid.data.input.InputRowSchema; import org.apache.druid.data.input.MapBasedInputRow; +import org.apache.druid.data.input.impl.MapInputRowParser; import org.apache.druid.data.input.kafka.KafkaRecordEntity; import org.apache.druid.indexing.seekablestream.SettableByteEntity; import org.apache.druid.java.util.common.CloseableIterators; import org.apache.druid.java.util.common.Pair; -import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.common.parsers.CloseableIterator; import org.apache.druid.java.util.common.parsers.ParseException; +import org.joda.time.DateTime; import javax.annotation.Nullable; - import java.io.IOException; +import java.util.AbstractMap; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; +import java.util.stream.Collectors; public class KafkaInputReader implements InputEntityReader { - private static final Logger log = new Logger(KafkaInputReader.class); - private final InputRowSchema inputRowSchema; private final SettableByteEntity source; private final Function headerParserSupplier; @@ -55,6 +56,7 @@ public class KafkaInputReader implements InputEntityReader private final InputEntityReader valueParser; private final String keyColumnName; private final String timestampColumnName; + private final String topicColumnName; /** * @@ -73,7 +75,8 @@ public KafkaInputReader( @Nullable Function keyParserSupplier, InputEntityReader valueParser, String keyColumnName, - String timestampColumnName + String timestampColumnName, + String topicColumnName ) { this.inputRowSchema = inputRowSchema; @@ -83,25 +86,92 @@ public KafkaInputReader( this.valueParser = valueParser; this.keyColumnName = keyColumnName; this.timestampColumnName = timestampColumnName; + this.topicColumnName = topicColumnName; + } + + @Override + public CloseableIterator read() throws IOException + { + final KafkaRecordEntity record = source.getEntity(); + final Map mergedHeaderMap = extractHeaderAndKeys(record); + + // Ignore tombstone records that have null values. + if (record.getRecord().value() != null) { + return buildBlendedRows(valueParser, mergedHeaderMap); + } else { + return CloseableIterators.withEmptyBaggage(buildInputRowsForMap(mergedHeaderMap).iterator()); + } } - private List getFinalDimensionList(HashSet newDimensions) + @Override + public CloseableIterator sample() throws IOException { - final List schemaDimensions = inputRowSchema.getDimensionsSpec().getDimensionNames(); - if (!schemaDimensions.isEmpty()) { - return schemaDimensions; + final KafkaRecordEntity record = source.getEntity(); + InputRowListPlusRawValues keysAndHeader = extractHeaderAndKeysSample(record); + if (record.getRecord().value() != null) { + return buildBlendedRowsSample(valueParser, keysAndHeader.getRawValues()); } else { - return Lists.newArrayList( - Sets.difference(newDimensions, inputRowSchema.getDimensionsSpec().getDimensionExclusions()) - ); + final List rows = Collections.singletonList(keysAndHeader); + return CloseableIterators.withEmptyBaggage(rows.iterator()); } } - private CloseableIterator buildBlendedRows(InputEntityReader valueParser, Map headerKeyList) throws IOException + private Map extractHeader(KafkaRecordEntity record) + { + final Map mergedHeaderMap = new HashMap<>(); + if (headerParserSupplier != null) { + KafkaHeaderReader headerParser = headerParserSupplier.apply(record); + List> headerList = headerParser.read(); + for (Pair ele : headerList) { + mergedHeaderMap.put(ele.lhs, ele.rhs); + } + } + + // Add kafka record timestamp to the mergelist, we will skip record timestamp if the same key exists already in + // the header list + mergedHeaderMap.putIfAbsent(timestampColumnName, record.getRecord().timestamp()); + + // Add kafka record topic to the mergelist, only if the key doesn't already exist + mergedHeaderMap.putIfAbsent(topicColumnName, record.getRecord().topic()); + + return mergedHeaderMap; + } + + private Map extractHeaderAndKeys(KafkaRecordEntity record) throws IOException + { + final Map mergedHeaderMap = extractHeader(record); + final InputEntityReader keyParser = (keyParserSupplier == null) ? null : keyParserSupplier.apply(record); + if (keyParser != null) { + try (CloseableIterator keyIterator = keyParser.read()) { + // Key currently only takes the first row and ignores the rest. + if (keyIterator.hasNext()) { + // Return type for the key parser should be of type MapBasedInputRow + // Parsers returning other types are not compatible currently. + MapBasedInputRow keyRow = (MapBasedInputRow) keyIterator.next(); + // Add the key to the mergeList only if the key string is not already present + mergedHeaderMap.putIfAbsent( + keyColumnName, + keyRow.getEvent().entrySet().stream().findFirst().get().getValue() + ); + } + } + catch (ClassCastException e) { + throw new IOException( + "Unsupported keyFormat. KafkaInputformat only supports input format that return MapBasedInputRow rows" + ); + } + } + return mergedHeaderMap; + } + + private CloseableIterator buildBlendedRows( + InputEntityReader valueParser, + Map headerKeyList + ) throws IOException { return valueParser.read().map( r -> { - MapBasedInputRow valueRow; + final MapBasedInputRow valueRow; try { // Return type for the value parser should be of type MapBasedInputRow // Parsers returning other types are not compatible currently. @@ -113,83 +183,177 @@ private CloseableIterator buildBlendedRows(InputEntityReader valuePars "Unsupported input format in valueFormat. KafkaInputFormat only supports input format that return MapBasedInputRow rows" ); } - Map event = new HashMap<>(headerKeyList); - /* Currently we prefer payload attributes if there is a collision in names. - We can change this beahvior in later changes with a config knob. This default - behavior lets easy porting of existing inputFormats to the new one without any changes. - */ - event.putAll(valueRow.getEvent()); - - HashSet newDimensions = new HashSet(valueRow.getDimensions()); + + final Map event = buildBlendedEventMap(valueRow.getEvent(), headerKeyList); + final HashSet newDimensions = new HashSet<>(valueRow.getDimensions()); newDimensions.addAll(headerKeyList.keySet()); // Remove the dummy timestamp added in KafkaInputFormat newDimensions.remove(KafkaInputFormat.DEFAULT_AUTO_TIMESTAMP_STRING); + final DateTime timestamp = MapInputRowParser.parseTimestamp(inputRowSchema.getTimestampSpec(), event); return new MapBasedInputRow( - inputRowSchema.getTimestampSpec().extractTimestamp(event), - getFinalDimensionList(newDimensions), + timestamp, + MapInputRowParser.findDimensions( + inputRowSchema.getTimestampSpec(), + inputRowSchema.getDimensionsSpec(), + newDimensions + ), event ); } ); } - private CloseableIterator buildRowsWithoutValuePayload(Map headerKeyList) + private InputRowListPlusRawValues extractHeaderAndKeysSample(KafkaRecordEntity record) throws IOException { - HashSet newDimensions = new HashSet(headerKeyList.keySet()); - InputRow row = new MapBasedInputRow( - inputRowSchema.getTimestampSpec().extractTimestamp(headerKeyList), - getFinalDimensionList(newDimensions), - headerKeyList - ); - List rows = Collections.singletonList(row); - return CloseableIterators.withEmptyBaggage(rows.iterator()); - } - - @Override - public CloseableIterator read() throws IOException - { - KafkaRecordEntity record = source.getEntity(); - Map mergeMap = new HashMap<>(); - if (headerParserSupplier != null) { - KafkaHeaderReader headerParser = headerParserSupplier.apply(record); - List> headerList = headerParser.read(); - for (Pair ele : headerList) { - mergeMap.put(ele.lhs, ele.rhs); - } - } - - // Add kafka record timestamp to the mergelist, we will skip record timestamp if the same key exists already in the header list - mergeMap.putIfAbsent(timestampColumnName, record.getRecord().timestamp()); - + Map mergedHeaderMap = extractHeader(record); InputEntityReader keyParser = (keyParserSupplier == null) ? null : keyParserSupplier.apply(record); if (keyParser != null) { - try (CloseableIterator keyIterator = keyParser.read()) { + try (CloseableIterator keyIterator = keyParser.sample()) { // Key currently only takes the first row and ignores the rest. if (keyIterator.hasNext()) { // Return type for the key parser should be of type MapBasedInputRow // Parsers returning other types are not compatible currently. - MapBasedInputRow keyRow = (MapBasedInputRow) keyIterator.next(); + InputRowListPlusRawValues keyRow = keyIterator.next(); // Add the key to the mergeList only if the key string is not already present - mergeMap.putIfAbsent(keyColumnName, keyRow.getEvent().entrySet().stream().findFirst().get().getValue()); + mergedHeaderMap.putIfAbsent( + keyColumnName, + keyRow.getRawValues().entrySet().stream().findFirst().get().getValue() + ); + return InputRowListPlusRawValues.of(buildInputRowsForMap(mergedHeaderMap), mergedHeaderMap); } } catch (ClassCastException e) { - throw new IOException("Unsupported input format in keyFormat. KafkaInputformat only supports input format that return MapBasedInputRow rows"); + throw new IOException( + "Unsupported keyFormat. KafkaInputformat only supports input format that return MapBasedInputRow rows" + ); } } + return InputRowListPlusRawValues.of(buildInputRowsForMap(mergedHeaderMap), mergedHeaderMap); + } - // Ignore tombstone records that have null values. - if (record.getRecord().value() != null) { - return buildBlendedRows(valueParser, mergeMap); - } else { - return buildRowsWithoutValuePayload(mergeMap); - } + private CloseableIterator buildBlendedRowsSample( + InputEntityReader valueParser, + Map headerKeyList + ) throws IOException + { + return valueParser.sample().map( + rowAndValues -> { + if (rowAndValues.getParseException() != null) { + return rowAndValues; + } + List newInputRows = Lists.newArrayListWithCapacity(rowAndValues.getInputRows().size()); + List> newRawRows = Lists.newArrayListWithCapacity(rowAndValues.getRawValues().size()); + ParseException parseException = null; + + for (Map raw : rowAndValues.getRawValuesList()) { + newRawRows.add(buildBlendedEventMap(raw, headerKeyList)); + } + for (InputRow r : rowAndValues.getInputRows()) { + MapBasedInputRow valueRow = null; + try { + valueRow = (MapBasedInputRow) r; + } + catch (ClassCastException e) { + parseException = new ParseException( + null, + "Unsupported input format in valueFormat. KafkaInputFormat only supports input format that return MapBasedInputRow rows" + ); + } + if (valueRow != null) { + final Map event = buildBlendedEventMap(valueRow.getEvent(), headerKeyList); + final HashSet newDimensions = new HashSet<>(valueRow.getDimensions()); + newDimensions.addAll(headerKeyList.keySet()); + // Remove the dummy timestamp added in KafkaInputFormat + newDimensions.remove(KafkaInputFormat.DEFAULT_AUTO_TIMESTAMP_STRING); + newInputRows.add( + new MapBasedInputRow( + inputRowSchema.getTimestampSpec().extractTimestamp(event), + MapInputRowParser.findDimensions( + inputRowSchema.getTimestampSpec(), + inputRowSchema.getDimensionsSpec(), + newDimensions + ), + event + ) + ); + } + } + return InputRowListPlusRawValues.ofList(newRawRows, newInputRows, parseException); + } + ); } - @Override - public CloseableIterator sample() throws IOException + private List buildInputRowsForMap(Map headerKeyList) { - return read().map(row -> InputRowListPlusRawValues.of(row, ((MapBasedInputRow) row).getEvent())); + return Collections.singletonList( + new MapBasedInputRow( + inputRowSchema.getTimestampSpec().extractTimestamp(headerKeyList), + MapInputRowParser.findDimensions( + inputRowSchema.getTimestampSpec(), + inputRowSchema.getDimensionsSpec(), + headerKeyList.keySet() + ), + headerKeyList + ) + ); + } + + /** + * Builds a map that blends two {@link Map}, presenting the combined keyset of both maps, and preferring to read + * from the first map and falling back to the second map if the value is not present. + * + * This strategy is used rather than just copying the values of the keyset into a new map so that any 'flattening' + * machinery (such as {@link Map} created by {@link org.apache.druid.java.util.common.parsers.ObjectFlatteners}) is + * still in place to be lazily evaluated instead of eagerly copying. + */ + private static Map buildBlendedEventMap(Map map, Map fallback) + { + final Set keySet = new HashSet<>(fallback.keySet()); + keySet.addAll(map.keySet()); + + return new AbstractMap() + { + @Override + public Object get(Object key) + { + return map.getOrDefault((String) key, fallback.get(key)); + } + + @Override + public Set keySet() + { + return keySet; + } + + @Override + public Set> entrySet() + { + return keySet().stream() + .map( + field -> new Entry() + { + @Override + public String getKey() + { + return field; + } + + @Override + public Object getValue() + { + return get(field); + } + + @Override + public Object setValue(final Object value) + { + throw new UnsupportedOperationException(); + } + } + ) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + }; } } diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/data/input/kafkainput/KafkaInputFormatTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/data/input/kafkainput/KafkaInputFormatTest.java index 9f7afe482a35..21a0550f53e7 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/data/input/kafkainput/KafkaInputFormatTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/data/input/kafkainput/KafkaInputFormatTest.java @@ -28,7 +28,6 @@ import org.apache.druid.data.input.InputEntityReader; import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRowSchema; -import org.apache.druid.data.input.MapBasedInputRow; import org.apache.druid.data.input.impl.DimensionsSpec; import org.apache.druid.data.input.impl.JsonInputFormat; import org.apache.druid.data.input.impl.TimestampSpec; @@ -251,6 +250,7 @@ public void testWithHeaderKeyAndValue() throws IOException Assert.assertEquals("4", Iterables.getOnlyElement(row.getDimension("root_baz"))); Assert.assertEquals("1", Iterables.getOnlyElement(row.getDimension("path_omg"))); Assert.assertEquals("1", Iterables.getOnlyElement(row.getDimension("jq_omg"))); + Assert.assertEquals(ImmutableMap.of("mg", 1L), row.getRaw("o")); // Header verification Assert.assertEquals("application/json", Iterables.getOnlyElement(row.getDimension("kafka.newheader.encoding"))); @@ -409,7 +409,6 @@ public byte[] value() while (iterator.hasNext()) { final InputRow row = iterator.next(); - final MapBasedInputRow mrow = (MapBasedInputRow) row; // Payload verifications // this isn't super realistic, since most of these columns are not actually defined in the dimensionSpec // but test reading them anyway since it isn't technically illegal @@ -420,6 +419,7 @@ public byte[] value() Assert.assertEquals("4", Iterables.getOnlyElement(row.getDimension("root_baz"))); Assert.assertEquals("1", Iterables.getOnlyElement(row.getDimension("path_omg"))); Assert.assertEquals("1", Iterables.getOnlyElement(row.getDimension("jq_omg"))); + Assert.assertEquals(ImmutableMap.of("mg", 1L), row.getRaw("o")); // Header verification Assert.assertEquals("application/json", Iterables.getOnlyElement(row.getDimension("kafka.newheader.encoding"))); @@ -526,6 +526,7 @@ public void testWithOutKeyAndHeaderSpecs() throws IOException Assert.assertEquals("4", Iterables.getOnlyElement(row.getDimension("root_baz"))); Assert.assertEquals("1", Iterables.getOnlyElement(row.getDimension("path_omg"))); Assert.assertEquals("1", Iterables.getOnlyElement(row.getDimension("jq_omg"))); + Assert.assertEquals(ImmutableMap.of("mg", 1L), row.getRaw("o")); numActualIterations++; } @@ -610,6 +611,7 @@ public void testWithMultipleMixedRecords() throws IOException Assert.assertEquals("4", Iterables.getOnlyElement(row.getDimension("root_baz"))); Assert.assertEquals("1", Iterables.getOnlyElement(row.getDimension("path_omg"))); Assert.assertEquals("1", Iterables.getOnlyElement(row.getDimension("jq_omg"))); + Assert.assertEquals(ImmutableMap.of("mg", 1L), row.getRaw("o")); Assert.assertEquals(String.valueOf(i), Iterables.getOnlyElement(row.getDimension("index"))); From 95efc4f56dd9f19375bc4b58123e832c733f1446 Mon Sep 17 00:00:00 2001 From: Pankaj kumar Date: Wed, 6 Dec 2023 14:36:47 +0530 Subject: [PATCH 102/114] Fix test cases --- .../src/main/java/org/apache/druid/storage/s3/S3Utils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java index a256b7001506..6209248cf49d 100644 --- a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java +++ b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java @@ -331,7 +331,7 @@ static void uploadFileIfPossible( putObjectRequest.setAccessControlList(S3Utils.grantFullControlToBucketOwner(service, bucket)); } log.info("Pushing [%s] to bucket[%s] and key[%s].", file, bucket, key); - service.upload(putObjectRequest); + service.putObject(putObjectRequest); } @Nullable From d62e7c72dac63d477b6aab124053afbb6c9f7e96 Mon Sep 17 00:00:00 2001 From: Pankaj kumar Date: Wed, 6 Dec 2023 15:38:05 +0530 Subject: [PATCH 103/114] Fix test cases --- .../java/org/apache/druid/storage/s3/S3Utils.java | 2 +- .../org/apache/druid/storage/s3/S3TaskLogsTest.java | 13 ++++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java index 6209248cf49d..a256b7001506 100644 --- a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java +++ b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java @@ -331,7 +331,7 @@ static void uploadFileIfPossible( putObjectRequest.setAccessControlList(S3Utils.grantFullControlToBucketOwner(service, bucket)); } log.info("Pushing [%s] to bucket[%s] and key[%s].", file, bucket, key); - service.putObject(putObjectRequest); + service.upload(putObjectRequest); } @Nullable diff --git a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3TaskLogsTest.java b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3TaskLogsTest.java index bc1c7e66cf1c..a434739ed21b 100644 --- a/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3TaskLogsTest.java +++ b/extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3TaskLogsTest.java @@ -29,7 +29,6 @@ import com.amazonaws.services.s3.model.Owner; import com.amazonaws.services.s3.model.Permission; import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.PutObjectResult; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.google.common.base.Optional; @@ -123,11 +122,9 @@ public void testTaskLogsPushWithAclEnabled() throws Exception } @Test - public void test_pushTaskStatus() throws IOException + public void test_pushTaskStatus() throws IOException, InterruptedException { - EasyMock.expect(s3Client.putObject(EasyMock.anyObject(PutObjectRequest.class))) - .andReturn(new PutObjectResult()) - .once(); + s3Client.upload(EasyMock.anyObject(PutObjectRequest.class)); EasyMock.replay(s3Client); @@ -148,12 +145,10 @@ public void test_pushTaskStatus() throws IOException } @Test - public void test_pushTaskPayload() throws IOException + public void test_pushTaskPayload() throws IOException, InterruptedException { Capture putObjectRequestCapture = Capture.newInstance(CaptureType.FIRST); - EasyMock.expect(s3Client.putObject(EasyMock.capture(putObjectRequestCapture))) - .andReturn(new PutObjectResult()) - .once(); + s3Client.upload(EasyMock.capture(putObjectRequestCapture)); EasyMock.replay(s3Client); From 21e5724a2b76148578b4e4a282d6766b27b53ffe Mon Sep 17 00:00:00 2001 From: Pankaj kumar Date: Wed, 6 Dec 2023 18:10:09 +0530 Subject: [PATCH 104/114] Fix test cases --- .../org/apache/druid/emitter/kafka/KafkaEmitterConfig.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitterConfig.java b/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitterConfig.java index 09deffcc81a7..5c6c9b75aa65 100644 --- a/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitterConfig.java +++ b/extensions-contrib/kafka-emitter/src/main/java/org/apache/druid/emitter/kafka/KafkaEmitterConfig.java @@ -58,7 +58,12 @@ public String toString() @JsonCreator public static EventType fromString(String name) { - return valueOf(StringUtils.toUpperCase(name)); + for (EventType eventType : EventType.values()) { + if (eventType.toString().equalsIgnoreCase(name)) { + return eventType; + } + } + throw new IllegalArgumentException("Invalid EventType value: " + name); } } From a98c3ba784f3798dd20d16476571b1b643005d40 Mon Sep 17 00:00:00 2001 From: Pankaj kumar Date: Sat, 9 Dec 2023 00:54:51 +0530 Subject: [PATCH 105/114] upgrade dependency as per druid 28 --- pom.xml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index 50b54254e130..63a63c963f28 100644 --- a/pom.xml +++ b/pom.xml @@ -79,16 +79,15 @@ 3.5.1 2.0.0 2.2.4 - 2.13.9 + 2.13.11 1.23.0 1.11.3 - + 1.35.0 4.2.0 - 2.0.0 + 2.2.0 10.14.2.0 4.2.19 2.20.0 @@ -815,7 +814,7 @@ org.xerial.snappy snappy-java - 1.1.8.4 + 1.1.10.4 com.google.protobuf From d97a5caef4b845dca49f4d3acda2a6d649082665 Mon Sep 17 00:00:00 2001 From: Pankaj kumar Date: Sun, 10 Dec 2023 23:32:30 +0530 Subject: [PATCH 106/114] Removing unnecessary change --- pom.xml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/pom.xml b/pom.xml index 63a63c963f28..36c80acd3502 100644 --- a/pom.xml +++ b/pom.xml @@ -2168,19 +2168,5 @@ true - - hadoop3 - - - hadoop3.enabled - true - - - - 3.3.1 - 5.3.6.Final - 4.5.13 - - From a2cf23f80c7213d22554535acb9e1a85bef6c0d4 Mon Sep 17 00:00:00 2001 From: pagrawal Date: Tue, 12 Dec 2023 18:55:06 +0530 Subject: [PATCH 107/114] Change Maven repository URL --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 36c80acd3502..ee8a03e99f84 100644 --- a/pom.xml +++ b/pom.xml @@ -130,9 +130,9 @@ v1-rev20230301-2.0.0 - apache.snapshots - Apache Snapshot Repository - https://repository.apache.org/snapshots + maven.org + Maven Central Repository + https://repo1.maven.org/maven2/ 0.19.0-alpha From 13f48b2360dc30b15cf58782794c821dac39b4d9 Mon Sep 17 00:00:00 2001 From: pagrawal Date: Tue, 12 Dec 2023 19:31:42 +0530 Subject: [PATCH 108/114] Add Druid.xml --- .idea/inspectionProfiles/Druid.xml | 508 +++++++++++++++++++++++++++++ 1 file changed, 508 insertions(+) create mode 100644 .idea/inspectionProfiles/Druid.xml diff --git a/.idea/inspectionProfiles/Druid.xml b/.idea/inspectionProfiles/Druid.xml new file mode 100644 index 000000000000..8847cd8a489a --- /dev/null +++ b/.idea/inspectionProfiles/Druid.xml @@ -0,0 +1,508 @@ + + + + From d00dac5736e85e9b84578900bfc711b2aa96ad06 Mon Sep 17 00:00:00 2001 From: pagrawal Date: Tue, 12 Dec 2023 20:25:15 +0530 Subject: [PATCH 109/114] Update tag name to match version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ee8a03e99f84..33c44b53b40e 100644 --- a/pom.xml +++ b/pom.xml @@ -66,7 +66,7 @@ scm:git:ssh://git@github.com/apache/druid.git scm:git:ssh://git@github.com/apache/druid.git https://github.com/apache/druid.git - druid-28.0.0 + 28.0.0-SNAPSHOT From f0b9016350732962ac5ee04c654457ff6f0d7c28 Mon Sep 17 00:00:00 2001 From: Keerthana Srikanth Date: Tue, 12 Dec 2023 13:15:32 +0530 Subject: [PATCH 110/114] Fix dist-used profile to use Hadoop compile version (#173) --- distribution/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/pom.xml b/distribution/pom.xml index 2a74e4469551..cc035e21a3e2 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -657,7 +657,7 @@ -l ${settings.localRepository} -h - org.apache.hadoop:hadoop-client:2.8.5 + org.apache.hadoop:hadoop-client:${hadoop.compile.version} -c org.apache.druid.extensions:druid-datasketches -c From da6ef5f1815f04de32ce0742651b9970d54f1392 Mon Sep 17 00:00:00 2001 From: pagrawal Date: Thu, 14 Dec 2023 18:15:22 +0530 Subject: [PATCH 111/114] Changes based on PR comments --- .idea/inspectionProfiles/Druid.xml | 2 +- .../java/org/apache/druid/emitter/kafka/KafkaEmitter.java | 6 +++--- .../org/apache/druid/emitter/kafka/KafkaEmitterTest.java | 4 ++-- .../src/components/refresh-button/refresh-button.tsx | 4 +++- web-console/src/views/tasks-view/tasks-view.tsx | 1 + 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.idea/inspectionProfiles/Druid.xml b/.idea/inspectionProfiles/Druid.xml index 8847cd8a489a..d1f72dd2763f 100644 --- a/.idea/inspectionProfiles/Druid.xml +++ b/.idea/inspectionProfiles/Druid.xml @@ -505,4 +505,4 @@