#40 Provide Optional Built-in Hadoop Dependencies

MANIFEST.in

@@ -18,6 +18,7 @@
 
 global-exclude *.py[cod] __pycache__ .DS_Store
 recursive-include deps/jars *.jar
+recursive-include deps/hadoop *.jar
 include README.md
 include LICENSE
 include NOTICE

dev/build-source-distribution-package.sh

@@ -19,20 +19,34 @@ CURR_DIR=`pwd`
 BASE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 PROJECT_ROOT="${BASE_DIR}/../"
 
-# prepare bridge jar
-
-DEPS_DIR=${PROJECT_ROOT}/deps/jars
+DEPS_DIR=${PROJECT_ROOT}/deps
 rm -rf ${DEPS_DIR}
-mkdir -p ${DEPS_DIR}
-touch ${DEPS_DIR}/__init__.py
+
+# prepare bridge jar
+BRIDGE_DEPS_DIR=${DEPS_DIR}/jars
+mkdir -p ${BRIDGE_DEPS_DIR}
+touch ${BRIDGE_DEPS_DIR}/__init__.py
 
 cd ${PROJECT_ROOT}/paimon-python-java-bridge
 
 # get bridge jar version
-JAR_VERSION=$(sed -n 's/.*<version>\(.*\)<\/version>.*/\1/p' pom.xml | head -n 1)
+BRIDGE_JAR_VERSION=$(sed -n 's/.*<version>\(.*\)<\/version>.*/\1/p' pom.xml | head -n 1)
+
+mvn clean install -DskipTests
+cp "target/paimon-python-java-bridge-${BRIDGE_JAR_VERSION}.jar" ${BRIDGE_DEPS_DIR}
+
+# prepare hadoop-deps jar
+HADOOP_DEPS_DIR=${DEPS_DIR}/hadoop
+mkdir -p ${HADOOP_DEPS_DIR}
+touch ${HADOOP_DEPS_DIR}/__init__.py
+
+cd ${PROJECT_ROOT}/hadoop-deps
+
+# get hadoop-deps jar version
+HADOOP_JAR_VERSION=$(sed -n 's/.*<version>\(.*\)<\/version>.*/\1/p' pom.xml | head -n 1)
 
 mvn clean install -DskipTests
-cp "target/paimon-python-java-bridge-${JAR_VERSION}.jar" ${DEPS_DIR}
+cp "target/hadoop-deps-${HADOOP_JAR_VERSION}.jar" ${HADOOP_DEPS_DIR}
 
 cd ${CURR_DIR}
 

dev/lint-python.sh

@@ -580,6 +580,8 @@ function tox_check() {
     # dummy jar needed by setup.py
     mkdir -p $PAIMON_PYTHON_DIR/deps/jars
     touch $PAIMON_PYTHON_DIR/deps/jars/dummy.jar
+    mkdir -p $PAIMON_PYTHON_DIR/deps/hadoop
+    touch $PAIMON_PYTHON_DIR/deps/hadoop/dummy.jar
 
     if [[ -n "$GITHUB_ACTION" ]]; then
         # Run tests in all versions triggered by a Git push (tests aren't so many currently)

hadoop-deps/pom.xml

@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.apache.pypaimon</groupId>
+    <artifactId>hadoop-deps</artifactId>
+    <version>3.3.4</version>
+
+    <properties>
+        <hadoop.version>3.3.4</hadoop.version>
+        <log4j.version>2.17.1</log4j.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <version>${hadoop.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-hdfs-client</artifactId>
+            <version>${hadoop.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-api</artifactId>
+            <version>${log4j.version}</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <createDependencyReducedPom>false</createDependencyReducedPom>
+                            <transformers>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                            </transformers>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+</project>

pypaimon/py4j/gateway_server.py

@@ -102,12 +102,20 @@ def _get_classpath(env):
     return os.pathsep.join(classpath)
 
 
+_HADOOP_DEPS_PACKAGE = 'pypaimon.hadoop-deps'
+
+
 def _get_hadoop_classpath(env):
     if constants.PYPAIMON_HADOOP_CLASSPATH in env:
         return env[constants.PYPAIMON_HADOOP_CLASSPATH]
-
-    if 'HADOOP_CLASSPATH' in env:
+    elif 'HADOOP_CLASSPATH' in env:
         return env['HADOOP_CLASSPATH']
     else:
-        raise EnvironmentError(f"You haven't set '{constants.PYPAIMON_HADOOP_CLASSPATH}', \
- and 'HADOOP_CLASSPATH' is also not set. Ensure one of them is set.")
+        # use built-in hadoop
+        jars = importlib.resources.files(_HADOOP_DEPS_PACKAGE)
+        one_jar = next(iter(jars.iterdir()), None)
+        if not one_jar:
+            raise EnvironmentError(f"The built-in Hadoop environment has been broken, this \
+            is unexpected. You can set one of '{constants.PYPAIMON_HADOOP_CLASSPATH}' or \
+            'HADOOP_CLASSPATH' to continue.")
+        return os.path.join(os.path.dirname(str(one_jar)), '*')

setup.py

@@ -38,7 +38,8 @@
     'pypaimon.api',
     'pypaimon.py4j',
     'pypaimon.py4j.util',
-    'pypaimon.jars'
+    'pypaimon.jars',
+    'pypaimon.hadoop-deps'
 ]
 
 install_requires = [
@@ -57,10 +58,12 @@
     include_package_data=True,
     # releasing tool will generate deps
     package_dir={
-        "pypaimon.jars": "deps/jars"
+        "pypaimon.jars": "deps/jars",
+        "pypaimon.hadoop-deps": "deps/hadoop"
     },
     package_data={
-        "pypaimon.jars": ["*.jar"]
+        "pypaimon.jars": ["*.jar"],
+        "pypaimon.hadoop-deps": ["*.jar"]
     },
     install_requires=install_requires,
     description='Apache Paimon Python API',

tools/releasing/create_source_release.sh

@@ -55,11 +55,13 @@ fi
 
 ###########################
 
-# prepare bridge jar
-
-DEPS_DIR=${PROJECT_ROOT}/deps/jars
+DEPS_DIR=${PROJECT_ROOT}/deps
 rm -rf ${DEPS_DIR}
-mkdir -p ${DEPS_DIR}
+
+# prepare bridge jar
+BRIDGE_DEPS_DIR=${DEPS_DIR}/jars
+mkdir -p ${BRIDGE_DEPS_DIR}
+touch ${BRIDGE_DEPS_DIR}/__init__.py
 
 cd ${PROJECT_ROOT}/paimon-python-java-bridge
 
@@ -70,10 +72,23 @@ if grep -q "<version>.*SNAPSHOT</version>" "pom.xml"; then
 fi
 
 # get bridge jar version
-JAR_VERSION=$(sed -n 's/.*<version>\(.*\)<\/version>.*/\1/p' pom.xml | head -n 1)
+BRIDGE_JAR_VERSION=$(sed -n 's/.*<version>\(.*\)<\/version>.*/\1/p' pom.xml | head -n 1)
+
+mvn clean install -DskipTests
+cp "target/paimon-python-java-bridge-${BRIDGE_JAR_VERSION}.jar" ${BRIDGE_DEPS_DIR}
+
+# prepare hadoop-deps jar
+HADOOP_DEPS_DIR=${DEPS_DIR}/hadoop
+mkdir -p ${HADOOP_DEPS_DIR}
+touch ${HADOOP_DEPS_DIR}/__init__.py
+
+cd ${PROJECT_ROOT}/hadoop-deps
+
+# get hadoop-deps jar version
+HADOOP_JAR_VERSION=$(sed -n 's/.*<version>\(.*\)<\/version>.*/\1/p' pom.xml | head -n 1)
 
 mvn clean install -DskipTests
-cp "target/paimon-python-java-bridge-${JAR_VERSION}.jar" ${DEPS_DIR}
+cp "target/hadoop-deps-${HADOOP_JAR_VERSION}.jar" ${HADOOP_DEPS_DIR}
 
 cd ${CURR_DIR}