[feature-wip](multicatalog)support apche druid catalog

be/src/vec/exec/vjdbc_connector.cpp

@@ -478,7 +478,8 @@ Status JdbcConnector::_check_type(SlotDescriptor* slot_desc, const std::string&
         break;
     }
     case TYPE_JSONB: {
-        if (type_str != "java.lang.String" && type_str != "org.postgresql.util.PGobject") {
+        if (type_str != "java.lang.String" && type_str != "org.postgresql.util.PGobject" &&
+            type_str != "java.lang.Object") {
             return Status::InternalError(error_msg);
         }
 

docker/thirdparties/docker-compose/druid/druid.env

@@ -0,0 +1,53 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Java tuning
+#DRUID_XMX=1g
+#DRUID_XMS=1g
+#DRUID_MAXNEWSIZE=250m
+#DRUID_NEWSIZE=250m
+#DRUID_MAXDIRECTMEMORYSIZE=6172m
+DRUID_SINGLE_NODE_CONF=nano-quickstart
+
+druid_emitter_logging_logLevel=debug
+
+druid_extensions_loadList=["druid-histogram", "druid-datasketches", "druid-lookups-cached-global", "postgresql-metadata-storage", "druid-multi-stage-query"]
+
+druid_zk_service_host=zookeeper-druid
+
+druid_metadata_storage_host=postgres-druid
+druid_metadata_storage_type=postgresql
+druid_metadata_storage_connector_connectURI=jdbc:postgresql://postgres-druid:5432/druid
+druid_metadata_storage_connector_user=druid
+druid_metadata_storage_connector_password=FoolishPassword
+
+druid_coordinator_balancer_strategy=cachingCost
+
+druid_indexer_runner_javaOptsArray=["-server", "-Xmx1g", "-Xms1g", "-XX:MaxDirectMemorySize=3g", "-Duser.timezone=UTC", "-Dfile.encoding=UTF-8", "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"]
+druid_indexer_fork_property_druid_processing_buffer_sizeBytes=256MiB
+
+druid_storage_type=local
+druid_storage_storageDirectory=/opt/shared/segments
+druid_indexer_logs_type=file
+druid_indexer_logs_directory=/opt/shared/indexing-logs
+
+druid_processing_numThreads=4
+druid_processing_numMergeBuffers=2
+
+DRUID_LOG4J=<?xml version="1.0" encoding="UTF-8" ?><Configuration status="WARN"><Appenders><Console name="Console" target="SYSTEM_OUT"><PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/></Console></Appenders><Loggers><Root level="info"><AppenderRef ref="Console"/></Root><Logger name="org.apache.druid.jetty.RequestLog" additivity="false" level="DEBUG"><AppenderRef ref="Console"/></Logger></Loggers></Configuration>

docker/thirdparties/docker-compose/druid/druid.yaml.tpl

@@ -0,0 +1,203 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+version: "3.9"
+
+volumes:
+  metadata_data: {}
+  middle_var: {}
+  historical_var: {}
+  broker_var: {}
+  coordinator_var: {}
+  router_var: {}
+  druid_shared: {}
+  zookeeper_data: {}
+
+services:
+  postgres-druid:
+    image: postgres:14
+    container_name: postgres-druid
+    ports:
+      - "11432:5432"
+    volumes:
+      - ./data/druid_pg_data:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_PASSWORD=FoolishPassword
+      - POSTGRES_USER=druid
+      - POSTGRES_DB=druid
+        # networks:
+        #   - doris--druid
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "druid"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  # Need 3.5 or later for container nodes
+  zookeeper-druid:
+    image: zookeeper:3.7.0
+    container_name: zookeeper-druid
+    volumes:
+      - ./data/druid_zk_data:/data
+    ports:
+      - "12181:2181"
+    environment:
+      - ZOO_MY_ID=1
+        # networks:
+        #   - doris--druid
+    healthcheck:
+      test: nc -z localhost 2181 || exit -1
+      start_period: 15s
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+  coordinator-druid:
+    image: apache/druid:27.0.0
+    container_name: coordinator-druid
+    volumes:
+      - ./data/druid_shared:/opt/shared
+      - ./data/druid_coordinator_var:/opt/druid/var
+    depends_on:
+      - zookeeper-druid
+      - postgres-druid
+    ports:
+      - "11881:8081"
+        # networks:
+        #   - doris--druid
+    healthcheck:
+      test: nc -z localhost 8081 || exit -1
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    command:
+      - coordinator
+    env_file:
+      - druid.env
+
+  broker-druid:
+    image: apache/druid:27.0.0
+    container_name: broker-druid
+    volumes:
+      - ./data/druid_broker_var:/opt/druid/var
+    depends_on:
+      - zookeeper-druid
+      - postgres-druid
+      - coordinator-druid
+        # networks:
+        #   - doris--druid
+    ports:
+      - "11882:8082"
+    healthcheck:
+      test: nc -z localhost 8082 || exit -1
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    command:
+      - broker
+    env_file:
+      - druid.env
+
+  historical-druid:
+    image: apache/druid:27.0.0
+    container_name: historical-druid
+    volumes:
+      - ./data/druid_shared:/opt/shared
+      - ./data/druid_historical_var:/opt/druid/var
+    depends_on:
+      - zookeeper-druid
+      - postgres-druid
+      - coordinator-druid
+    ports:
+      - "11883:8083"
+        # networks:
+        #   - doris--druid
+    healthcheck:
+      test: nc -z localhost 8083 || exit -1
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    command:
+      - historical
+    env_file:
+      - druid.env
+
+  middlemanager-druid:
+    image: apache/druid:27.0.0
+    container_name: middlemanager-druid
+    volumes:
+      - ./data/druid_shared:/opt/shared
+      - ./data/druid_middle_var:/opt/druid/var
+    depends_on:
+      - zookeeper-druid
+      - postgres-druid
+      - coordinator-druid
+    ports:
+      - "11891:8091"
+      - "21100-21105:8100-8105"
+        # networks:
+        #   - doris--druid
+    healthcheck:
+      test: nc -z localhost 8091 || exit -1
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    command:
+      - middleManager
+    env_file:
+      - druid.env
+
+  router-druid:
+    image: apache/druid:27.0.0
+    container_name: router-druid
+    volumes:
+      - ./data/druid_router_var:/opt/druid/var
+    depends_on:
+      - zookeeper-druid
+      - postgres-druid
+      - coordinator-druid
+        # networks:
+        #   - doris--druid
+    ports:
+      - "21888:8888"
+    healthcheck:
+      test: nc -z localhost 8888 || exit -1
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    command:
+      - router
+    env_file:
+      - druid.env
+
+  toolbox-druid:
+    image: webdevops/toolbox
+    container_name: toolbox-druid
+    volumes:
+      - ./scripts:/mnt/scripts
+    environment:
+      DRUID_COORDINATOR_HOST: "coordinator-druid"
+    command: [ "sh","-c","/mnt/scripts/druid_init.sh" ]
+    depends_on:
+      - postgres-druid
+      - zookeeper-druid
+      - middlemanager-druid
+      - broker-druid
+      - historical-druid
+      - router-druid
+      - coordinator-druid

docker/thirdparties/docker-compose/druid/scripts/data/kttm.json

@@ -0,0 +1,74 @@
+{
+  "type": "index_parallel",
+  "spec": {
+    "ioConfig": {
+      "type": "index_parallel",
+      "inputSource": {
+        "type": "inline",
+        "data": "{\"session\":\"S56194838\",\"number\":\"16\",\"event\":\"{\\\"type\\\":\\\"PercentClear\\\",\\\"percentage\\\":55}\",\"agent\":\"{\\\"type\\\":\\\"Browser\\\",\\\"category\\\":\\\"Personal computer\\\",\\\"browser\\\":\\\"Chrome\\\",\\\"browser_version\\\":\\\"76.0.3809.100\\\",\\\"os\\\":\\\"Windows 7\\\",\\\"platform\\\":\\\"Windows\\\"}\",\"client_ip\":\"181.13.41.82\",\"geo_ip\":\"{\\\"continent\\\":\\\"South America\\\",\\\"country\\\":\\\"Argentina\\\",\\\"region\\\":\\\"Santa Fe\\\",\\\"city\\\":\\\"Rosario\\\"}\",\"language\":\"[\\\"es\\\",\\\"es-419\\\"]\",\"adblock_list\":\"NoAdblock\",\"app_version\":\"1.9.6\",\"path\":\"http://www.koalastothemax.com/\",\"loaded_image\":\"http://www.koalastothemax.com/img/koalas2.jpg\",\"referrer\":\"Direct\",\"referrer_host\":\"Direct\",\"server_ip\":\"172.31.57.89\",\"screen\":\"1680x1050\",\"window\":\"1680x939\",\"session_length\":76261,\"timezone\":\"N/A\",\"timezone_offset\":\"180\"}\n{\"session\":\"S46093731\",\"number\":\"24\",\"event\":\"{\\\"type\\\":\\\"PercentClear\\\",\\\"percentage\\\":85}\",\"agent\":\"{\\\"type\\\":\\\"Mobile Browser\\\",\\\"category\\\":\\\"Smartphone\\\",\\\"browser\\\":\\\"Chrome Mobile\\\",\\\"browser_version\\\":\\\"50.0.2661.89\\\",\\\"os\\\":\\\"Android\\\",\\\"platform\\\":\\\"Android\\\"}\",\"client_ip\":\"177.242.100.0\",\"geo_ip\":\"{\\\"continent\\\":\\\"North America\\\",\\\"country\\\":\\\"Mexico\\\",\\\"region\\\":\\\"Chihuahua\\\",\\\"city\\\":\\\"Nuevo Casas Grandes\\\"}\",\"language\":\"[\\\"en\\\",\\\"es\\\",\\\"es-419\\\",\\\"es-MX\\\"]\",\"adblock_list\":\"NoAdblock\",\"app_version\":\"1.9.6\",\"path\":\"https://koalastothemax.com/\",\"loaded_image\":\"https://koalastothemax.com/img/koalas1.jpg\",\"referrer\":\"https://www.google.com/\",\"referrer_host\":\"www.google.com\",\"server_ip\":\"172.31.11.5\",\"screen\":\"320x570\",\"window\":\"540x743\",\"session_length\":252689,\"timezone\":\"CDT\",\"timezone_offset\":\"300\"}\n{\"session\":\"S13352079\",\"number\":\"24\",\"event\":\"{\\\"type\\\":\\\"PercentClear\\\",\\\"percentage\\\":90}\",\"agent\":\"{\\\"type\\\":\\\"Browser\\\",\\\"category\\\":\\\"Personal computer\\\",\\\"browser\\\":\\\"Chrome\\\",\\\"browser_version\\\":\\\"76.0.3809.100\\\",\\\"os\\\":\\\"Windows 7\\\",\\\"platform\\\":\\\"Windows\\\"}\",\"client_ip\":\"181.46.136.44\",\"geo_ip\":\"{\\\"continent\\\":\\\"South America\\\",\\\"country\\\":\\\"Argentina\\\",\\\"region\\\":\\\"Buenos Aires\\\",\\\"city\\\":\\\"Luis Guillon\\\"}\",\"language\":\"[\\\"en\\\",\\\"es\\\",\\\"es-419\\\",\\\"es-US\\\"]\",\"adblock_list\":\"NoAdblock\",\"app_version\":\"1.9.6\",\"path\":\"http://www.koalastothemax.com/\",\"loaded_image\":\"http://www.koalastothemax.com/img/koalas.jpg\",\"referrer\":\"Direct\",\"referrer_host\":\"Direct\",\"server_ip\":\"172.31.11.5\",\"screen\":\"1366x768\",\"window\":\"1366x652\",\"session_length\":1753602,\"timezone\":\"N/A\",\"timezone_offset\":\"180\"}\n{\"session\":\"S28264557\",\"number\":\"15\",\"event\":\"{\\\"type\\\":\\\"PercentClear\\\",\\\"percentage\\\":55}\",\"agent\":\"{\\\"type\\\":\\\"Browser\\\",\\\"category\\\":\\\"Personal computer\\\",\\\"browser\\\":\\\"Opera\\\",\\\"browser_version\\\":\\\"62.0.3331.116\\\",\\\"os\\\":\\\"Windows\\\",\\\"platform\\\":\\\"Windows\\\"}\",\"client_ip\":\"71.82.190.9\",\"geo_ip\":\"{\\\"continent\\\":\\\"North America\\\",\\\"country\\\":\\\"United States\\\",\\\"region\\\":\\\"Michigan\\\",\\\"city\\\":\\\"Allegan\\\"}\",\"language\":\"[\\\"en\\\",\\\"en-US\\\"]\",\"adblock_list\":\"NoAdblock\",\"app_version\":\"1.9.6\",\"path\":\"http://koalastothemax.com/\",\"loaded_image\":\"http://koalastothemax.com/img/koalas3.jpg\",\"referrer\":\"Direct\",\"referrer_host\":\"Direct\",\"server_ip\":\"172.31.57.89\",\"screen\":\"1920x1080\",\"window\":\"1880x976\",\"session_length\":62093,\"timezone\":\"N/A\",\"timezone_offset\":\"240\"}\n{\"session\":\"S81338885\",\"number\":\"18\",\"event\":\"{\\\"type\\\":\\\"PercentClear\\\",\\\"percentage\\\":60}\",\"agent\":\"{\\\"type\\\":\\\"Mobile Browser\\\",\\\"category\\\":\\\"Smartphone\\\",\\\"browser\\\":\\\"Mobile Safari\\\",\\\"browser_version\\\":\\\"12.0\\\",\\\"os\\\":\\\"iOS\\\",\\\"platform\\\":\\\"iOS\\\"}\",\"client_ip\":\"100.0.162.244\",\"geo_ip\":\"{\\\"continent\\\":\\\"North America\\\",\\\"country\\\":\\\"United States\\\",\\\"region\\\":\\\"Rhode Island\\\",\\\"city\\\":\\\"Woonsocket\\\"}\",\"language\":\"en-us\",\"adblock_list\":\"NoAdblock\",\"app_version\":\"1.9.6\",\"path\":\"http://koalastothemax.com/\",\"loaded_image\":\"http://koalastothemax.com/img/koalas2.jpg\",\"referrer\":\"Direct\",\"referrer_host\":\"Direct\",\"server_ip\":\"172.31.11.5\",\"screen\":\"375x667\",\"window\":\"540x796\",\"session_length\":79764,\"timezone\":\"EDT\",\"timezone_offset\":\"240\"}\n{\"session\":\"S18516388\",\"number\":\"4\",\"event\":\"{\\\"type\\\":\\\"LayerClear\\\",\\\"layer\\\":4}\",\"agent\":\"{\\\"type\\\":\\\"Browser\\\",\\\"category\\\":\\\"Personal computer\\\",\\\"browser\\\":\\\"Chrome\\\",\\\"browser_version\\\":\\\"76.0.3809.100\\\",\\\"os\\\":\\\"Windows\\\",\\\"platform\\\":\\\"Windows\\\"}\",\"client_ip\":\"86.124.118.86\",\"geo_ip\":\"{\\\"continent\\\":\\\"Europe\\\",\\\"country\\\":\\\"Romania\\\",\\\"region\\\":\\\"Judetul Braila\\\",\\\"city\\\":\\\"Braila\\\"}\",\"language\":\"[\\\"en\\\",\\\"en-US\\\",\\\"ro\\\",\\\"ro-RO\\\"]\",\"adblock_list\":\"EasyList\",\"app_version\":\"1.9.6\",\"path\":\"https://koalastothemax.com/\",\"loaded_image\":\"https://koalastothemax.com/img/koalas2.jpg\",\"referrer\":\"https://www.google.com/\",\"referrer_host\":\"www.google.com\",\"server_ip\":\"172.31.57.89\",\"screen\":\"1366x768\",\"window\":\"1366x657\",\"session_length\":6264,\"timezone\":\"N/A\",\"timezone_offset\":\"-180\"}\n{\"session\":\"S93504612\",\"number\":\"1\",\"event\":\"{\\\"type\\\":\\\"LayerClear\\\",\\\"layer\\\":6}\",\"agent\":\"{\\\"type\\\":\\\"Browser\\\",\\\"category\\\":\\\"Personal computer\\\",\\\"browser\\\":\\\"Chrome\\\",\\\"browser_version\\\":\\\"76.0.3809.100\\\",\\\"os\\\":\\\"Windows\\\",\\\"platform\\\":\\\"Windows\\\"}\",\"client_ip\":\"68.82.4.9\",\"geo_ip\":\"{\\\"continent\\\":\\\"North America\\\",\\\"country\\\":\\\"United States\\\",\\\"region\\\":\\\"Pennsylvania\\\",\\\"city\\\":\\\"Philadelphia\\\"}\",\"language\":\"[\\\"en\\\",\\\"en-US\\\"]\",\"adblock_list\":\"EasyList\",\"app_version\":\"1.9.6\",\"path\":\"https://koalastothemax.com/\",\"loaded_image\":\"https://koalastothemax.com/img/koalas3.jpg\",\"referrer\":\"https://www.google.com/\",\"referrer_host\":\"www.google.com\",\"server_ip\":\"172.31.11.5\",\"screen\":\"1366x768\",\"window\":\"1366x657\",\"session_length\":8070,\"timezone\":\"N/A\",\"timezone_offset\":\"240\"}\n{\"session\":\"S93504612\",\"number\":\"2\",\"event\":\"{\\\"type\\\":\\\"PercentClear\\\",\\\"percentage\\\":0}\",\"agent\":\"{\\\"type\\\":\\\"Browser\\\",\\\"category\\\":\\\"Personal computer\\\",\\\"browser\\\":\\\"Chrome\\\",\\\"browser_version\\\":\\\"76.0.3809.100\\\",\\\"os\\\":\\\"Windows\\\",\\\"platform\\\":\\\"Windows\\\"}\",\"client_ip\":\"68.82.4.9\",\"geo_ip\":\"{\\\"continent\\\":\\\"North America\\\",\\\"country\\\":\\\"United States\\\",\\\"region\\\":\\\"Pennsylvania\\\",\\\"city\\\":\\\"Philadelphia\\\"}\",\"language\":\"[\\\"en\\\",\\\"en-US\\\"]\",\"adblock_list\":\"EasyList\",\"app_version\":\"1.9.6\",\"path\":\"https://koalastothemax.com/\",\"loaded_image\":\"https://koalastothemax.com/img/koalas3.jpg\",\"referrer\":\"https://www.google.com/\",\"referrer_host\":\"www.google.com\",\"server_ip\":\"172.31.11.5\",\"screen\":\"1366x768\",\"window\":\"1366x657\",\"session_length\":8078,\"timezone\":\"N/A\",\"timezone_offset\":\"240\"}\n{\"session\":\"S10902483\",\"number\":\"6\",\"event\":\"{\\\"type\\\":\\\"PercentClear\\\",\\\"percentage\\\":10}\",\"agent\":\"{\\\"type\\\":\\\"Mobile Browser\\\",\\\"category\\\":\\\"Smartphone\\\",\\\"browser\\\":\\\"Mobile Safari\\\",\\\"browser_version\\\":\\\"12.1.2\\\",\\\"os\\\":\\\"iOS\\\",\\\"platform\\\":\\\"iOS\\\"}\",\"client_ip\":\"72.93.52.73\",\"geo_ip\":\"{\\\"continent\\\":\\\"North America\\\",\\\"country\\\":\\\"United States\\\",\\\"region\\\":\\\"Massachusetts\\\",\\\"city\\\":\\\"Worcester\\\"}\",\"language\":\"en-us\",\"adblock_list\":\"NoAdblock\",\"app_version\":\"1.9.6\",\"path\":\"https://koalastothemax.com/\",\"loaded_image\":\"https://koalastothemax.com/img/koalas2.jpg\",\"referrer\":\"https://www.google.com/\",\"referrer_host\":\"www.google.com\",\"server_ip\":\"172.31.57.89\",\"screen\":\"414x736\",\"window\":\"540x811\",\"session_length\":49275,\"timezone\":\"EDT\",\"timezone_offset\":\"240\"}\n{\"session\":\"S81338885\",\"number\":\"19\",\"event\":\"{\\\"type\\\":\\\"PercentClear\\\",\\\"percentage\\\":65}\",\"agent\":\"{\\\"type\\\":\\\"Mobile Browser\\\",\\\"category\\\":\\\"Smartphone\\\",\\\"browser\\\":\\\"Mobile Safari\\\",\\\"browser_version\\\":\\\"12.0\\\",\\\"os\\\":\\\"iOS\\\",\\\"platform\\\":\\\"iOS\\\"}\",\"client_ip\":\"100.0.162.244\",\"geo_ip\":\"{\\\"continent\\\":\\\"North America\\\",\\\"country\\\":\\\"United States\\\",\\\"region\\\":\\\"Rhode Island\\\",\\\"city\\\":\\\"Woonsocket\\\"}\",\"language\":\"en-us\",\"adblock_list\":\"NoAdblock\",\"app_version\":\"1.9.6\",\"path\":\"http://koalastothemax.com/\",\"loaded_image\":\"http://koalastothemax.com/img/koalas2.jpg\",\"referrer\":\"Direct\",\"referrer_host\":\"Direct\",\"server_ip\":\"172.31.11.5\",\"screen\":\"375x667\",\"window\":\"540x796\",\"session_length\":82064,\"timezone\":\"EDT\",\"timezone_offset\":\"240\"}"
+      },
+      "inputFormat": {
+        "type": "json"
+      }
+    },
+    "tuningConfig": {
+      "type": "index_parallel",
+      "partitionsSpec": {
+        "type": "dynamic"
+      }
+    },
+    "dataSchema": {
+      "dataSource": "kttm_druid",
+      "timestampSpec": {
+        "column": "timestamp",
+        "format": "iso"
+      },
+      "dimensionsSpec": {
+        "dimensions": [
+          "session",
+          {
+            "name": "number",
+            "type": "float"
+          },
+          {
+            "name": "event",
+            "type": "json"
+          },
+          {
+            "name": "agent",
+            "type": "json"
+          },
+          "client_ip",
+          {
+            "name": "geo_ip",
+            "type": "json"
+          },
+          "language",
+          "adblock_list",
+          "app_version",
+          "path",
+          "loaded_image",
+          "referrer",
+          "referrer_host",
+          "server_ip",
+          "screen",
+          "window",
+          {
+            "type": "long",
+            "name": "session_length"
+          },
+          "timezone",
+          {
+            "name": "timezone_offset",
+            "type": "double"
+          }
+        ]
+      },
+      "granularitySpec": {
+        "queryGranularity": "none",
+        "rollup": false,
+        "segmentGranularity": "day"
+      }
+    }
+  }
+}