[build][sql] Make pulsar-trino-disto self-contained (apache#17062)

tisonkun · web-flow · commit 743b03807852 · 2022-09-09T20:01:46.000+02:00
diff --git a/.github/workflows/ci-owasp-dep-check.yaml b/.github/workflows/ci-owasp-dep-check.yaml
@@ -87,7 +87,7 @@ jobs:
           docker rmi $(docker images -q) -f
           df -h
 
-      # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan.
+      # Projects dependent on flume, hdfs, hbase, and trino currently excluded from the scan.
       - name: run "clean verify" to trigger dependency check
         if: ${{ steps.changes.outputs.poms == 'true' }}
         run: mvn -q -B -ntp clean verify -PskipDocker,owasp-dependency-check -DskipTests -pl '!pulsar-sql,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb'
diff --git a/bin/pulsar b/bin/pulsar
@@ -33,7 +33,6 @@ DEFAULT_PROXY_CONF=$PULSAR_HOME/conf/proxy.conf
 DEFAULT_STANDALONE_CONF=$PULSAR_HOME/conf/standalone.conf
 DEFAULT_WEBSOCKET_CONF=$PULSAR_HOME/conf/websocket.conf
 DEFAULT_LOG_CONF=$PULSAR_HOME/conf/log4j2.yaml
-DEFAULT_PULSAR_PRESTO_CONF=${PULSAR_HOME}/conf/presto
 DEFAULT_FUNCTIONS_LOG_CONF=$PULSAR_HOME/conf/functions_log4j2.xml
 
 # functions related variables
@@ -46,7 +45,8 @@ PY_INSTANCE_FILE=${PULSAR_PY_INSTANCE_FILE:-"${DEFAULT_PY_INSTANCE_FILE}"}
 DEFAULT_FUNCTIONS_EXTRA_DEPS_DIR=$PULSAR_HOME/instances/deps
 FUNCTIONS_EXTRA_DEPS_DIR=${PULSAR_FUNCTIONS_EXTRA_DEPS_DIR:-"${DEFAULT_FUNCTIONS_EXTRA_DEPS_DIR}"}
 SQL_HOME=$PULSAR_HOME/pulsar-sql
-PRESTO_HOME=${PULSAR_HOME}/lib/presto
+TRINO_HOME=${PULSAR_HOME}/trino
+DEFAULT_PULSAR_TRINO_CONF=${TRINO_HOME}/conf
 
 pulsar_help() {
     cat <<EOF
@@ -90,7 +90,7 @@ Environment variables:
    PULSAR_PROXY_CONF             Configuration file for Pulsar proxy (default: $DEFAULT_PROXY_CONF)
    PULSAR_WORKER_CONF            Configuration file for functions worker (default: $DEFAULT_WORKER_CONF)
    PULSAR_STANDALONE_CONF        Configuration file for standalone (default: $DEFAULT_STANDALONE_CONF)
-   PULSAR_PRESTO_CONF            Configuration directory for Pulsar Presto (default: $DEFAULT_PULSAR_PRESTO_CONF)
+   PULSAR_TRINO_CONF            Configuration directory for Pulsar SQL (default: $DEFAULT_PULSAR_TRINO_CONF)
    PULSAR_EXTRA_OPTS             Extra options to be passed to the jvm
    PULSAR_EXTRA_CLASSPATH        Add extra paths to the pulsar classpath
    PULSAR_PID_DIR                Folder where the pulsar server PID file should be stored
@@ -177,17 +177,17 @@ if [ ! -f "${PY_INSTANCE_FILE}" ]; then
     PY_INSTANCE_FILE=${BUILT_PY_INSTANCE_FILE}
 fi
 
-# find pulsar sql presto distribution location
-check_presto_libraries() {
-    if [ ! -d "${PRESTO_HOME}" ]; then
-
-        BUILT_PRESTO_HOME="${SQL_HOME}/presto-distribution/target/pulsar-presto-distribution"
-        if [ ! -d "${BUILT_PRESTO_HOME}" ]; then
-            echo "\nCouldn't find presto distribution.";
+# find pulsar sql trino distribution location
+check_trino_libraries() {
+    if [ ! -d "${TRINO_HOME}" ]; then
+        BUILT_TRINO_HOME="${SQL_HOME}/presto-distribution/target/pulsar-presto-distribution"
+        if [ ! -d "${BUILT_TRINO_HOME}" ]; then
+            echo "\nCouldn't find trino distribution.";
             echo "Make sure you've run 'mvn package'\n";
             exit 1;
         fi
-        PRESTO_HOME=${BUILT_PRESTO_HOME}
+        TRINO_HOME=${BUILT_TRINO_HOME}
+        PULSAR_TRINO_CONF=${BUILT_TRINO_HOME}/conf
     fi
 }
 
@@ -257,8 +257,19 @@ if [ -z "$PULSAR_LOG_CONF" ]; then
     PULSAR_LOG_CONF=$DEFAULT_LOG_CONF
 fi
 
-if [ -z "$PULSAR_PRESTO_CONF" ]; then
-    PULSAR_PRESTO_CONF=$DEFAULT_PULSAR_PRESTO_CONF
+if [ -z "$PULSAR_TRINO_CONF" ]; then
+    # TODO: As PIP-200 accepted, this compatibility is not promised. Refactor when we drop this b/w compatibility.
+    if [ -z "$PULSAR_PRESTO_CONF" ]; then
+      PULSAR_TRINO_CONF=$DEFAULT_PULSAR_TRINO_CONF
+    else
+      PULSAR_TRINO_CONF=$PULSAR_PRESTO_CONF
+    fi
+    if [ ! -d "${PULSAR_TRINO_CONF}" ]; then
+      FALLBACK_PULSAR_PRESTO_CONF=${PULSAR_HOME}/conf/presto
+      if [ -d "${FALLBACK_PULSAR_PRESTO_CONF}" ]; then
+        PULSAR_TRINO_CONF=$FALLBACK_PULSAR_PRESTO_CONF
+      fi
+    fi
 fi
 
 if [ -z "$FUNCTIONS_LOG_CONF"]; then
@@ -387,11 +398,11 @@ elif [ $COMMAND == "broker-tool" ]; then
 elif [ $COMMAND == "compact-topic" ]; then
     exec $JAVA $OPTS org.apache.pulsar.compaction.CompactorTool --broker-conf $PULSAR_BROKER_CONF $@
 elif [ $COMMAND == "sql" ]; then
-    check_presto_libraries
-    exec $JAVA -cp "${PRESTO_HOME}/lib/*" io.trino.cli.Trino --server localhost:8081 "${@}"
+    check_trino_libraries
+    exec $JAVA -cp "${TRINO_HOME}/lib/*" io.trino.cli.Trino --server localhost:8081 "${@}"
 elif [ $COMMAND == "sql-worker" ]; then
-    check_presto_libraries
-    exec python3 ${PRESTO_HOME}/bin/launcher.py --etc-dir ${PULSAR_PRESTO_CONF} "${@}"
+    check_trino_libraries
+    exec python3 ${TRINO_HOME}/bin/launcher.py --etc-dir ${PULSAR_TRINO_CONF} "${@}"
 elif [ $COMMAND == "tokens" ]; then
     exec $JAVA $OPTS org.apache.pulsar.utils.auth.tokens.TokensCliUtils $@
 elif [ $COMMAND == "version" ]; then
diff --git a/distribution/server/src/assemble/bin.xml b/distribution/server/src/assemble/bin.xml
@@ -62,15 +62,15 @@
     </fileSet>
     <fileSet>
       <directory>${basedir}/../../pulsar-sql/presto-distribution/target/pulsar-presto-distribution</directory>
-      <outputDirectory>lib/presto</outputDirectory>
+      <outputDirectory>trino</outputDirectory>
       <excludes>
         <exclude>bin</exclude>
         <exclude>bin/**</exclude>
       </excludes>
     </fileSet>
     <fileSet>
       <directory>${basedir}/../../pulsar-sql/presto-distribution/target/pulsar-presto-distribution</directory>
-      <outputDirectory>lib/presto</outputDirectory>
+      <outputDirectory>trino</outputDirectory>
       <includes>
         <include>bin/**</include>
       </includes>
diff --git a/docker-compose/kitchen-sink/docker-compose.yml b/docker-compose/kitchen-sink/docker-compose.yml
@@ -368,7 +368,7 @@ services:
     image: apachepulsar/pulsar-all:latest
     restart: on-failure
     command: >
-      bash -c "bin/apply-config-from-env-with-prefix.py SQL_PREFIX_ conf/presto/catalog/pulsar.properties && \
+      bash -c "bin/apply-config-from-env-with-prefix.py SQL_PREFIX_ trino/conf/catalog/pulsar.properties && \
                bin/apply-config-from-env.py conf/pulsar_env.sh && \
                bin/watch-znode.py -z $$zookeeperServers -p /initialized-$$clusterName -w && \
                exec bin/pulsar sql-worker run"
diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile
@@ -42,9 +42,9 @@ RUN for SUBDIRECTORY in conf data download logs; do \
      chmod -R g+w /pulsar/$SUBDIRECTORY; \
      done
 
-# Presto writes logs to this directory (at least during tests), so we need to give the process permission
-# to create those log directories. This should be removed when presto is removed.
-RUN chmod g+w /pulsar/lib/presto
+# Trino writes logs to this directory (at least during tests), so we need to give the process permission
+# to create those log directories. This should be removed when Trino is removed.
+RUN chmod g+w /pulsar/trino
 
 ### Create 2nd stage from Ubuntu image
 ### and add OpenJDK and Python dependencies (for Pulsar functions)
diff --git a/pulsar-sql/presto-distribution/src/assembly/assembly.xml b/pulsar-sql/presto-distribution/src/assembly/assembly.xml
@@ -46,6 +46,10 @@
             <directory>${basedir}/../presto-pulsar-plugin/target/pulsar-presto-connector/</directory>
             <outputDirectory>plugin/</outputDirectory>
         </fileSet>
+        <fileSet>
+            <directory>${basedir}/src/main/resources/conf/</directory>
+            <outputDirectory>conf/</outputDirectory>
+        </fileSet>
     </fileSets>
     <dependencySets>
         <dependencySet>
diff --git a/pulsar-sql/presto-distribution/src/main/resources/conf/catalog/pulsar.properties b/pulsar-sql/presto-distribution/src/main/resources/conf/catalog/pulsar.properties
diff --git a/pulsar-sql/presto-distribution/src/main/resources/conf/config.properties b/pulsar-sql/presto-distribution/src/main/resources/conf/config.properties
diff --git a/pulsar-sql/presto-distribution/src/main/resources/conf/jvm.config b/pulsar-sql/presto-distribution/src/main/resources/conf/jvm.config
diff --git a/pulsar-sql/presto-distribution/src/main/resources/conf/log.properties b/pulsar-sql/presto-distribution/src/main/resources/conf/log.properties
diff --git a/src/check-binary-license.sh b/src/check-binary-license.sh
@@ -41,7 +41,7 @@ if [ -z $TARBALL ]; then
     exit 1
 fi
 
-JARS=$(tar -tf $TARBALL | grep '\.jar' | grep -v 'lib/presto/' | grep -v '/examples/' | grep -v '/instances/' | sed 's!.*/!!' | sort)
+JARS=$(tar -tf $TARBALL | grep '\.jar' | grep -v 'trino/' | grep -v '/examples/' | grep -v '/instances/' | sed 's!.*/!!' | sort)
 
 LICENSEPATH=$(tar -tf $TARBALL  | awk '/^[^\/]*\/LICENSE/')
 LICENSE=$(tar -O -xf $TARBALL "$LICENSEPATH")
@@ -96,7 +96,7 @@ done
 
 if [ "$NO_PRESTO" -ne 1 ]; then
   # check pulsar sql jars
-  JARS=$(tar -tf $TARBALL | grep '\.jar' | grep 'lib/presto/' | grep -v pulsar-client | grep -v bouncy-castle-bc | grep -v pulsar-metadata | grep -v 'managed-ledger' | grep -v  'pulsar-client-admin' | grep -v  'pulsar-client-api' | grep -v 'pulsar-functions-api' | grep -v 'pulsar-presto-connector-original' | grep -v 'pulsar-presto-distribution' | grep -v 'pulsar-common' | grep -v 'pulsar-functions-proto' | grep -v 'pulsar-functions-utils' | grep -v 'pulsar-io-core' | grep -v 'pulsar-transaction-common' | grep -v 'pulsar-package-core' | sed 's!.*/!!' | sort)
+  JARS=$(tar -tf $TARBALL | grep '\.jar' | grep 'trino/' | grep -v pulsar-client | grep -v bouncy-castle-bc | grep -v pulsar-metadata | grep -v 'managed-ledger' | grep -v  'pulsar-client-admin' | grep -v  'pulsar-client-api' | grep -v 'pulsar-functions-api' | grep -v 'pulsar-presto-connector-original' | grep -v 'pulsar-presto-distribution' | grep -v 'pulsar-common' | grep -v 'pulsar-functions-proto' | grep -v 'pulsar-functions-utils' | grep -v 'pulsar-io-core' | grep -v 'pulsar-transaction-common' | grep -v 'pulsar-package-core' | sed 's!.*/!!' | sort)
   if [ -n "$JARS" ]; then
     LICENSEPATH=$(tar -tf $TARBALL  | awk '/^[^\/]*\/lib\/presto\/LICENSE/')
     LICENSE=$(tar -O -xf $TARBALL "$LICENSEPATH")
@@ -111,7 +111,7 @@ if [ "$NO_PRESTO" -ne 1 ]; then
 
         echo "$LICENSE" | grep -q $J
         if [ $? != 0 ]; then
-      echo $J unaccounted for in lib/presto/LICENSE
+      echo $J unaccounted for in trino/LICENSE
       EXIT=1
         fi
     done
@@ -120,7 +120,7 @@ if [ "$NO_PRESTO" -ne 1 ]; then
     for J in $LICENSEJARS; do
         echo "$JARS" | grep -q $J
         if [ $? != 0 ]; then
-      echo $J mentioned in lib/presto/LICENSE, but not bundled
+      echo $J mentioned in trino/LICENSE, but not bundled
       EXIT=2
         fi
     done
diff --git a/tests/docker-images/latest-version-image/Dockerfile b/tests/docker-images/latest-version-image/Dockerfile
@@ -89,7 +89,7 @@ COPY scripts/init-cluster.sh scripts/run-global-zk.sh scripts/run-local-zk.sh \
      scripts/run-standalone.sh \
      /pulsar/bin/
 
-COPY conf/presto/jvm.config /pulsar/conf/presto/
+COPY conf/presto/jvm.config /pulsar/trino/conf
 
 # copy python test examples
 RUN mkdir -p /pulsar/instances/deps
diff --git a/tests/docker-images/latest-version-image/scripts/run-presto-worker.sh b/tests/docker-images/latest-version-image/scripts/run-presto-worker.sh
@@ -18,7 +18,7 @@
 # under the License.
 #
 
-bin/apply-config-from-env-with-prefix.py SQL_PREFIX_ conf/presto/catalog/pulsar.properties && \
+bin/apply-config-from-env-with-prefix.py SQL_PREFIX_ trino/conf/catalog/pulsar.properties && \
     bin/apply-config-from-env.py conf/pulsar_env.sh
 
 if [ -z "$NO_AUTOSTART" ]; then
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PrestoWorkerContainer.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/containers/PrestoWorkerContainer.java
@@ -43,11 +43,11 @@ public PrestoWorkerContainer(String clusterName, String hostname) {
     @Override
     protected void afterStart() {
         DockerUtils.runCommandAsyncWithLogging(this.dockerClient, this.getContainerId(),
-                "tail", "-f", "/pulsar/lib/presto/var/log/launcher.log");
+                "tail", "-f", "/pulsar/trino/var/log/launcher.log");
         DockerUtils.runCommandAsyncWithLogging(this.dockerClient, this.getContainerId(),
                 "tail", "-f", "/var/log/pulsar/presto_worker.log");
         DockerUtils.runCommandAsyncWithLogging(this.dockerClient, this.getContainerId(),
-                "tail", "-f", "/pulsar/lib/presto/var/log/server.log");
+                "tail", "-f", "/pulsar/trino/var/log/server.log");
     }
 
     @Override
@@ -57,7 +57,7 @@ protected void beforeStop() {
             DockerUtils.dumpContainerDirToTargetCompressed(
                     getDockerClient(),
                     getContainerId(),
-                    "/pulsar/lib/presto/var/log"
+                    "/pulsar/trino/var/log"
             );
         }
     }
diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/topologies/PulsarCluster.java
@@ -438,7 +438,7 @@ private PrestoWorkerContainer buildPrestoWorkerContainer(String hostName, boolea
                 .withEnv("pulsar.web-service-url", "http://pulsar-broker-0:8080")
                 .withEnv("SQL_PREFIX_pulsar.max-message-size", "" + spec.maxMessageSize)
                 .withClasspathResourceMapping(
-                        resourcePath, "/pulsar/conf/presto/config.properties", BindMode.READ_WRITE);
+                        resourcePath, "/pulsar/trino/conf/config.properties", BindMode.READ_WRITE);
         if (spec.queryLastMessage) {
             container.withEnv("pulsar.bookkeeper-use-v2-protocol", "false")
                     .withEnv("pulsar.bookkeeper-explicit-interval", "10");

Original file line number	Diff line number	Diff line change
`@@ -18,7 +18,7 @@`
`18`	`18`	`# under the License.`
`19`	`19`	`#`
`20`	`20`
`21`		`-bin/apply-config-from-env-with-prefix.py SQL_PREFIX_ conf/presto/catalog/pulsar.properties && \`
	`21`	`+bin/apply-config-from-env-with-prefix.py SQL_PREFIX_ trino/conf/catalog/pulsar.properties && \`
`22`	`22`	`bin/apply-config-from-env.py conf/pulsar_env.sh`
`23`	`23`
`24`	`24`	`if [ -z "$NO_AUTOSTART" ]; then`