From 4444330901077b76a8f1d365677bd75dc8805c9c Mon Sep 17 00:00:00 2001 From: meteorgan Date: Mon, 25 Nov 2024 10:16:30 +0800 Subject: [PATCH] chore: add setup action for hadoop to avoid build failures (#5353) --- .github/actions/setup-hadoop/action.yaml | 60 +++++++++++++++++++ .github/services/hdfs/hdfs_cluster/action.yml | 16 +---- .../action.yml | 16 +---- .github/services/hdfs/hdfs_default/action.yml | 19 ++---- .../action.yml | 19 ++---- 5 files changed, 74 insertions(+), 56 deletions(-) create mode 100644 .github/actions/setup-hadoop/action.yaml diff --git a/.github/actions/setup-hadoop/action.yaml b/.github/actions/setup-hadoop/action.yaml new file mode 100644 index 000000000000..835371e42c5c --- /dev/null +++ b/.github/actions/setup-hadoop/action.yaml @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Setup Hadoop +description: 'Prepare hadoop binary and env' + +inputs: + need-hadoop: + description: "This setup needs hadoop or not" + +runs: + using: "composite" + steps: + - name: Setup java env + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "11" + + - name: Cache hadoop + id: cache-hadoop + uses: actions/cache@v4 + if: inputs.need-hadoop == 'true' + with: + path: /home/runner/hadoop-3.3.5 + key: cache-hadoop-3.3.5 + + - name: Build hadoop if not cached + if: steps.cache-hadoop.outputs.cache-hit != 'true' && inputs.need-hadoop == 'true' + shell: bash + run: | + set -e + curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner + + - name: Setup hadoop env + shell: bash + run: | + export HADOOP_HOME=/home/runner/hadoop-3.3.5 + echo "HADOOP_HOME=${HADOOP_HOME}" >> $GITHUB_ENV + echo "CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${HADOOP_HOME}/lib/native" >> $GITHUB_ENV + cp ${{ github.workspace }}/fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml + + + + diff --git a/.github/services/hdfs/hdfs_cluster/action.yml b/.github/services/hdfs/hdfs_cluster/action.yml index f54624264b0a..2047c65ff32c 100644 --- a/.github/services/hdfs/hdfs_cluster/action.yml +++ b/.github/services/hdfs/hdfs_cluster/action.yml @@ -25,20 +25,10 @@ runs: shell: bash working-directory: fixtures/hdfs run: docker compose -f docker-compose-hdfs-cluster.yml up -d --wait - - name: Setup java env - uses: actions/setup-java@v4 + - name: Setup hadoop + uses: ./.github/actions/setup-hadoop with: - distribution: temurin - java-version: "11" - - name: Setup hadoop env - shell: bash - run: | - curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner - export HADOOP_HOME=/home/runner/hadoop-3.3.5 - echo "HADOOP_HOME=${HADOOP_HOME}" >> $GITHUB_ENV - echo "CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${HADOOP_HOME}/lib/native" >> $GITHUB_ENV - cp ${{ github.workspace }}/fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml + need-hadoop: true - name: Setup opendal env shell: bash run: | diff --git a/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml b/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml index 860b6137a147..6ad8894b0a30 100644 --- a/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml +++ b/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml @@ -25,20 +25,10 @@ runs: shell: bash working-directory: fixtures/hdfs run: docker compose -f docker-compose-hdfs-cluster.yml up -d --wait - - name: Setup java env - uses: actions/setup-java@v4 + - name: Setup hadoop + uses: ./.github/actions/setup-hadoop with: - distribution: temurin - java-version: "11" - - name: Setup hadoop env - shell: bash - run: | - curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner - export HADOOP_HOME=/home/runner/hadoop-3.3.5 - echo "HADOOP_HOME=${HADOOP_HOME}" >> $GITHUB_ENV - echo "CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${HADOOP_HOME}/lib/native" >> $GITHUB_ENV - cp ${{ github.workspace }}/fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml + need-hadoop: true - name: Setup opendal env shell: bash run: | diff --git a/.github/services/hdfs/hdfs_default/action.yml b/.github/services/hdfs/hdfs_default/action.yml index d4bdce0765d1..a6e2540dd76a 100644 --- a/.github/services/hdfs/hdfs_default/action.yml +++ b/.github/services/hdfs/hdfs_default/action.yml @@ -21,25 +21,14 @@ description: 'Behavior test for hdfs default' runs: using: "composite" steps: - - name: Setup java env - uses: actions/setup-java@v4 + - name: Setup hadoop + uses: ./.github/actions/setup-hadoop with: - distribution: temurin - java-version: "11" - - name: Setup + need-hadoop: true + - name: Setup OpenDAL env shell: bash run: | - curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner - - export HADOOP_HOME="/home/runner/hadoop-3.3.5" - export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob) - - cp ./fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml - cat << EOF >> $GITHUB_ENV - HADOOP_HOME=${HADOOP_HOME} - CLASSPATH=${CLASSPATH} - LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native OPENDAL_HDFS_ROOT=/tmp/opendal/ OPENDAL_HDFS_NAME_NODE=default OPENDAL_HDFS_ENABLE_APPEND=false diff --git a/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml b/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml index b8de8671611a..57372570ff30 100644 --- a/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml +++ b/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml @@ -21,25 +21,14 @@ description: 'Behavior test for hdfs default with atomic write dir' runs: using: "composite" steps: - - name: Setup java env - uses: actions/setup-java@v4 + - name: Setup hadoop + uses: ./.github/actions/setup-hadoop with: - distribution: temurin - java-version: "11" - - name: Setup + need-hadoop: true + - name: Setup OpenDAL env shell: bash run: | - curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner - - export HADOOP_HOME="/home/runner/hadoop-3.3.5" - export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob) - - cp ./fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml - cat << EOF >> $GITHUB_ENV - HADOOP_HOME=${HADOOP_HOME} - CLASSPATH=${CLASSPATH} - LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native OPENDAL_HDFS_ROOT=/tmp/opendal/ OPENDAL_HDFS_ATOMIC_WRITE_DIR=/tmp/atomic_write_dir/opendal/ OPENDAL_HDFS_NAME_NODE=default