From 6f732e15a5489fb8ab42a6b07c899290fb857cbb Mon Sep 17 00:00:00 2001 From: meteorgan Date: Fri, 22 Nov 2024 23:16:19 +0800 Subject: [PATCH 1/3] chore: cache hadoop-3.3.5 to avoid build failures --- .github/actions/setup/action.yaml | 17 +++++++++++++++++ .github/services/hdfs/hdfs_cluster/action.yml | 1 - .../action.yml | 1 - .github/services/hdfs/hdfs_default/action.yml | 2 -- .../services/hdfs/hdfs_default_gcs/action.yml | 2 -- .../hdfs_default_on_azurite_azblob/action.yml | 2 -- .../hdfs/hdfs_default_on_minio_s3/action.yml | 2 -- .../action.yml | 2 -- .../workflows/test_behavior_binding_nodejs.yml | 1 + .../workflows/test_behavior_binding_python.yml | 1 + .github/workflows/test_behavior_core.yml | 1 + 11 files changed, 20 insertions(+), 12 deletions(-) diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml index 652b8a493bcd..b19588f7d3aa 100644 --- a/.github/actions/setup/action.yaml +++ b/.github/actions/setup/action.yaml @@ -31,6 +31,8 @@ inputs: github-token: description: "Github Token" default: "" + need-hadoop: + description: "This setup needs hadoop or not" runs: using: "composite" @@ -152,3 +154,18 @@ runs: rm foundationdb-clients_7.1.17-1_amd64.deb rm foundationdb-server_7.1.17-1_amd64.deb + - name: Cache hadoop + id: cache-hadoop + uses: actions/cache@v4 + if: inputs.need-hadoop == 'true' + with: + path: /home/runner/hadoop-3.3.5 + key: r0-hadoop-3.3.5 + + - name: Build hadoop if not cached + if: steps.cache-hadoop.outputs.cache-hit != 'true' && inputs.need-hadoop == 'true' + shell: bash + run: | + set -e + curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner + diff --git a/.github/services/hdfs/hdfs_cluster/action.yml b/.github/services/hdfs/hdfs_cluster/action.yml index f54624264b0a..099bb86761b5 100644 --- a/.github/services/hdfs/hdfs_cluster/action.yml +++ b/.github/services/hdfs/hdfs_cluster/action.yml @@ -33,7 +33,6 @@ runs: - name: Setup hadoop env shell: bash run: | - curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner export HADOOP_HOME=/home/runner/hadoop-3.3.5 echo "HADOOP_HOME=${HADOOP_HOME}" >> $GITHUB_ENV echo "CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)" >> $GITHUB_ENV diff --git a/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml b/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml index 860b6137a147..4742d86a5bdd 100644 --- a/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml +++ b/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml @@ -33,7 +33,6 @@ runs: - name: Setup hadoop env shell: bash run: | - curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner export HADOOP_HOME=/home/runner/hadoop-3.3.5 echo "HADOOP_HOME=${HADOOP_HOME}" >> $GITHUB_ENV echo "CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)" >> $GITHUB_ENV diff --git a/.github/services/hdfs/hdfs_default/action.yml b/.github/services/hdfs/hdfs_default/action.yml index d4bdce0765d1..0f855196434e 100644 --- a/.github/services/hdfs/hdfs_default/action.yml +++ b/.github/services/hdfs/hdfs_default/action.yml @@ -29,8 +29,6 @@ runs: - name: Setup shell: bash run: | - curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner - export HADOOP_HOME="/home/runner/hadoop-3.3.5" export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob) diff --git a/.github/services/hdfs/hdfs_default_gcs/action.yml b/.github/services/hdfs/hdfs_default_gcs/action.yml index 69aabeba75d8..1e644e009490 100644 --- a/.github/services/hdfs/hdfs_default_gcs/action.yml +++ b/.github/services/hdfs/hdfs_default_gcs/action.yml @@ -37,8 +37,6 @@ runs: - name: Setup shell: bash run: | - curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner - export HADOOP_HOME="/home/runner/hadoop-3.3.5" curl -LsSf -o ${HADOOP_HOME}/share/hadoop/common/lib/gcs-connector-hadoop3-2.2.19-shaded.jar https://github.com/GoogleCloudDataproc/hadoop-connectors/releases/download/v2.2.19/gcs-connector-hadoop3-2.2.19-shaded.jar diff --git a/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml b/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml index 94a2763af763..5e62dcb9de24 100644 --- a/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml +++ b/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml @@ -43,8 +43,6 @@ runs: OPENDAL_AZBLOB_ACCOUNT_NAME=devstoreaccount1 OPENDAL_AZBLOB_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== - curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner - export HADOOP_HOME="/home/runner/hadoop-3.3.5" curl -LsSf -o ${HADOOP_HOME}/share/hadoop/common/lib/hadoop-azure-3.3.5.jar https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-azure/3.3.5/hadoop-azure-3.3.5.jar diff --git a/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml b/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml index e75b8e415627..7853083dcdfe 100644 --- a/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml +++ b/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml @@ -40,8 +40,6 @@ runs: - name: Setup shell: bash run: | - curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner - export HADOOP_HOME="/home/runner/hadoop-3.3.5" curl -LsSf -o ${HADOOP_HOME}/share/hadoop/common/lib/hadoop-aws-3.3.5.jar https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.5/hadoop-aws-3.3.5.jar diff --git a/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml b/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml index b8de8671611a..2f1190ecf9a9 100644 --- a/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml +++ b/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml @@ -29,8 +29,6 @@ runs: - name: Setup shell: bash run: | - curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner - export HADOOP_HOME="/home/runner/hadoop-3.3.5" export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob) diff --git a/.github/workflows/test_behavior_binding_nodejs.yml b/.github/workflows/test_behavior_binding_nodejs.yml index 510fbe40401e..191420cfd238 100644 --- a/.github/workflows/test_behavior_binding_nodejs.yml +++ b/.github/workflows/test_behavior_binding_nodejs.yml @@ -43,6 +43,7 @@ jobs: need-nextest: true need-protoc: true need-rocksdb: true + need-hadoop: true github-token: ${{ secrets.GITHUB_TOKEN }} # TODO: 1password is only supported on linux diff --git a/.github/workflows/test_behavior_binding_python.yml b/.github/workflows/test_behavior_binding_python.yml index 8f3d4bbaf48a..bc4a5fc6794b 100644 --- a/.github/workflows/test_behavior_binding_python.yml +++ b/.github/workflows/test_behavior_binding_python.yml @@ -43,6 +43,7 @@ jobs: need-nextest: true need-protoc: true need-rocksdb: true + need-hadoop: true github-token: ${{ secrets.GITHUB_TOKEN }} # TODO: 1password is only supported on linux diff --git a/.github/workflows/test_behavior_core.yml b/.github/workflows/test_behavior_core.yml index ef49723c5ed1..08212f134840 100644 --- a/.github/workflows/test_behavior_core.yml +++ b/.github/workflows/test_behavior_core.yml @@ -42,6 +42,7 @@ jobs: with: need-protoc: true need-rocksdb: true + need-hadoop: true github-token: ${{ secrets.GITHUB_TOKEN }} # TODO: 1password is only supported on linux From 24d7f8b6926dd2b6adbfef25b6644aac982592cb Mon Sep 17 00:00:00 2001 From: meteorgan Date: Sat, 23 Nov 2024 23:06:33 +0800 Subject: [PATCH 2/3] chore: add setup action for hadoop --- .github/actions/setup-hadoop/action.yaml | 46 +++++++++++++++++++ .github/actions/setup/action.yaml | 17 ------- .github/services/hdfs/hdfs_cluster/action.yml | 4 ++ .../action.yml | 1 + .github/services/hdfs/hdfs_default/action.yml | 2 + .../services/hdfs/hdfs_default_gcs/action.yml | 2 + .../hdfs_default_on_azurite_azblob/action.yml | 2 + .../hdfs/hdfs_default_on_minio_s3/action.yml | 2 + .../action.yml | 2 + .../test_behavior_binding_nodejs.yml | 1 - .../test_behavior_binding_python.yml | 1 - .github/workflows/test_behavior_core.yml | 1 - 12 files changed, 61 insertions(+), 20 deletions(-) create mode 100644 .github/actions/setup-hadoop/action.yaml diff --git a/.github/actions/setup-hadoop/action.yaml b/.github/actions/setup-hadoop/action.yaml new file mode 100644 index 000000000000..e1ee5cd9f63c --- /dev/null +++ b/.github/actions/setup-hadoop/action.yaml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Setup Hadoop +description: 'Prepare Hadoop' + +inputs: + need-hadoop: + description: "This setup needs hadoop or not" + +runs: + using: "composite" + steps: + - name: Cache hadoop + id: cache-hadoop + uses: actions/cache@v4 + if: inputs.need-hadoop == 'true' + with: + path: /home/runner/hadoop-3.3.5 + key: cache-hadoop-3.3.5 + + - name: Build hadoop if not cached + if: steps.cache-hadoop.outputs.cache-hit != 'true' && inputs.need-hadoop == 'true' + shell: bash + run: | + set -e + curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner + + + + + diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml index b19588f7d3aa..652b8a493bcd 100644 --- a/.github/actions/setup/action.yaml +++ b/.github/actions/setup/action.yaml @@ -31,8 +31,6 @@ inputs: github-token: description: "Github Token" default: "" - need-hadoop: - description: "This setup needs hadoop or not" runs: using: "composite" @@ -154,18 +152,3 @@ runs: rm foundationdb-clients_7.1.17-1_amd64.deb rm foundationdb-server_7.1.17-1_amd64.deb - - name: Cache hadoop - id: cache-hadoop - uses: actions/cache@v4 - if: inputs.need-hadoop == 'true' - with: - path: /home/runner/hadoop-3.3.5 - key: r0-hadoop-3.3.5 - - - name: Build hadoop if not cached - if: steps.cache-hadoop.outputs.cache-hit != 'true' && inputs.need-hadoop == 'true' - shell: bash - run: | - set -e - curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner - diff --git a/.github/services/hdfs/hdfs_cluster/action.yml b/.github/services/hdfs/hdfs_cluster/action.yml index 099bb86761b5..eae84b234725 100644 --- a/.github/services/hdfs/hdfs_cluster/action.yml +++ b/.github/services/hdfs/hdfs_cluster/action.yml @@ -30,6 +30,10 @@ runs: with: distribution: temurin java-version: "11" + - name: Setup hadoop + uses: ./.github/actions/setup-hadoop + with: + need-hadoop: true - name: Setup hadoop env shell: bash run: | diff --git a/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml b/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml index 4742d86a5bdd..860b6137a147 100644 --- a/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml +++ b/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml @@ -33,6 +33,7 @@ runs: - name: Setup hadoop env shell: bash run: | + curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner export HADOOP_HOME=/home/runner/hadoop-3.3.5 echo "HADOOP_HOME=${HADOOP_HOME}" >> $GITHUB_ENV echo "CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)" >> $GITHUB_ENV diff --git a/.github/services/hdfs/hdfs_default/action.yml b/.github/services/hdfs/hdfs_default/action.yml index 0f855196434e..d4bdce0765d1 100644 --- a/.github/services/hdfs/hdfs_default/action.yml +++ b/.github/services/hdfs/hdfs_default/action.yml @@ -29,6 +29,8 @@ runs: - name: Setup shell: bash run: | + curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner + export HADOOP_HOME="/home/runner/hadoop-3.3.5" export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob) diff --git a/.github/services/hdfs/hdfs_default_gcs/action.yml b/.github/services/hdfs/hdfs_default_gcs/action.yml index 1e644e009490..69aabeba75d8 100644 --- a/.github/services/hdfs/hdfs_default_gcs/action.yml +++ b/.github/services/hdfs/hdfs_default_gcs/action.yml @@ -37,6 +37,8 @@ runs: - name: Setup shell: bash run: | + curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner + export HADOOP_HOME="/home/runner/hadoop-3.3.5" curl -LsSf -o ${HADOOP_HOME}/share/hadoop/common/lib/gcs-connector-hadoop3-2.2.19-shaded.jar https://github.com/GoogleCloudDataproc/hadoop-connectors/releases/download/v2.2.19/gcs-connector-hadoop3-2.2.19-shaded.jar diff --git a/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml b/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml index 5e62dcb9de24..94a2763af763 100644 --- a/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml +++ b/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml @@ -43,6 +43,8 @@ runs: OPENDAL_AZBLOB_ACCOUNT_NAME=devstoreaccount1 OPENDAL_AZBLOB_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== + curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner + export HADOOP_HOME="/home/runner/hadoop-3.3.5" curl -LsSf -o ${HADOOP_HOME}/share/hadoop/common/lib/hadoop-azure-3.3.5.jar https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-azure/3.3.5/hadoop-azure-3.3.5.jar diff --git a/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml b/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml index 7853083dcdfe..e75b8e415627 100644 --- a/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml +++ b/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml @@ -40,6 +40,8 @@ runs: - name: Setup shell: bash run: | + curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner + export HADOOP_HOME="/home/runner/hadoop-3.3.5" curl -LsSf -o ${HADOOP_HOME}/share/hadoop/common/lib/hadoop-aws-3.3.5.jar https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.5/hadoop-aws-3.3.5.jar diff --git a/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml b/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml index 2f1190ecf9a9..b8de8671611a 100644 --- a/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml +++ b/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml @@ -29,6 +29,8 @@ runs: - name: Setup shell: bash run: | + curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner + export HADOOP_HOME="/home/runner/hadoop-3.3.5" export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob) diff --git a/.github/workflows/test_behavior_binding_nodejs.yml b/.github/workflows/test_behavior_binding_nodejs.yml index 191420cfd238..510fbe40401e 100644 --- a/.github/workflows/test_behavior_binding_nodejs.yml +++ b/.github/workflows/test_behavior_binding_nodejs.yml @@ -43,7 +43,6 @@ jobs: need-nextest: true need-protoc: true need-rocksdb: true - need-hadoop: true github-token: ${{ secrets.GITHUB_TOKEN }} # TODO: 1password is only supported on linux diff --git a/.github/workflows/test_behavior_binding_python.yml b/.github/workflows/test_behavior_binding_python.yml index bc4a5fc6794b..8f3d4bbaf48a 100644 --- a/.github/workflows/test_behavior_binding_python.yml +++ b/.github/workflows/test_behavior_binding_python.yml @@ -43,7 +43,6 @@ jobs: need-nextest: true need-protoc: true need-rocksdb: true - need-hadoop: true github-token: ${{ secrets.GITHUB_TOKEN }} # TODO: 1password is only supported on linux diff --git a/.github/workflows/test_behavior_core.yml b/.github/workflows/test_behavior_core.yml index 08212f134840..ef49723c5ed1 100644 --- a/.github/workflows/test_behavior_core.yml +++ b/.github/workflows/test_behavior_core.yml @@ -42,7 +42,6 @@ jobs: with: need-protoc: true need-rocksdb: true - need-hadoop: true github-token: ${{ secrets.GITHUB_TOKEN }} # TODO: 1password is only supported on linux From 194c9c124a16c899410a5edd3ff87f91de2fcc3c Mon Sep 17 00:00:00 2001 From: meteorgan Date: Sat, 23 Nov 2024 23:17:48 +0800 Subject: [PATCH 3/3] chore: move java setup && hadoop env to hadoop setup --- .github/actions/setup-hadoop/action.yaml | 16 +++++++++++++++- .github/services/hdfs/hdfs_cluster/action.yml | 13 ------------- .../action.yml | 16 +++------------- .github/services/hdfs/hdfs_default/action.yml | 19 ++++--------------- .../action.yml | 19 ++++--------------- 5 files changed, 26 insertions(+), 57 deletions(-) diff --git a/.github/actions/setup-hadoop/action.yaml b/.github/actions/setup-hadoop/action.yaml index e1ee5cd9f63c..835371e42c5c 100644 --- a/.github/actions/setup-hadoop/action.yaml +++ b/.github/actions/setup-hadoop/action.yaml @@ -16,7 +16,7 @@ # under the License. name: Setup Hadoop -description: 'Prepare Hadoop' +description: 'Prepare hadoop binary and env' inputs: need-hadoop: @@ -25,6 +25,12 @@ inputs: runs: using: "composite" steps: + - name: Setup java env + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "11" + - name: Cache hadoop id: cache-hadoop uses: actions/cache@v4 @@ -40,6 +46,14 @@ runs: set -e curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner + - name: Setup hadoop env + shell: bash + run: | + export HADOOP_HOME=/home/runner/hadoop-3.3.5 + echo "HADOOP_HOME=${HADOOP_HOME}" >> $GITHUB_ENV + echo "CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${HADOOP_HOME}/lib/native" >> $GITHUB_ENV + cp ${{ github.workspace }}/fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml diff --git a/.github/services/hdfs/hdfs_cluster/action.yml b/.github/services/hdfs/hdfs_cluster/action.yml index eae84b234725..2047c65ff32c 100644 --- a/.github/services/hdfs/hdfs_cluster/action.yml +++ b/.github/services/hdfs/hdfs_cluster/action.yml @@ -25,23 +25,10 @@ runs: shell: bash working-directory: fixtures/hdfs run: docker compose -f docker-compose-hdfs-cluster.yml up -d --wait - - name: Setup java env - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: "11" - name: Setup hadoop uses: ./.github/actions/setup-hadoop with: need-hadoop: true - - name: Setup hadoop env - shell: bash - run: | - export HADOOP_HOME=/home/runner/hadoop-3.3.5 - echo "HADOOP_HOME=${HADOOP_HOME}" >> $GITHUB_ENV - echo "CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${HADOOP_HOME}/lib/native" >> $GITHUB_ENV - cp ${{ github.workspace }}/fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml - name: Setup opendal env shell: bash run: | diff --git a/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml b/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml index 860b6137a147..6ad8894b0a30 100644 --- a/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml +++ b/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml @@ -25,20 +25,10 @@ runs: shell: bash working-directory: fixtures/hdfs run: docker compose -f docker-compose-hdfs-cluster.yml up -d --wait - - name: Setup java env - uses: actions/setup-java@v4 + - name: Setup hadoop + uses: ./.github/actions/setup-hadoop with: - distribution: temurin - java-version: "11" - - name: Setup hadoop env - shell: bash - run: | - curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner - export HADOOP_HOME=/home/runner/hadoop-3.3.5 - echo "HADOOP_HOME=${HADOOP_HOME}" >> $GITHUB_ENV - echo "CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob)" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${HADOOP_HOME}/lib/native" >> $GITHUB_ENV - cp ${{ github.workspace }}/fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml + need-hadoop: true - name: Setup opendal env shell: bash run: | diff --git a/.github/services/hdfs/hdfs_default/action.yml b/.github/services/hdfs/hdfs_default/action.yml index d4bdce0765d1..a6e2540dd76a 100644 --- a/.github/services/hdfs/hdfs_default/action.yml +++ b/.github/services/hdfs/hdfs_default/action.yml @@ -21,25 +21,14 @@ description: 'Behavior test for hdfs default' runs: using: "composite" steps: - - name: Setup java env - uses: actions/setup-java@v4 + - name: Setup hadoop + uses: ./.github/actions/setup-hadoop with: - distribution: temurin - java-version: "11" - - name: Setup + need-hadoop: true + - name: Setup OpenDAL env shell: bash run: | - curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner - - export HADOOP_HOME="/home/runner/hadoop-3.3.5" - export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob) - - cp ./fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml - cat << EOF >> $GITHUB_ENV - HADOOP_HOME=${HADOOP_HOME} - CLASSPATH=${CLASSPATH} - LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native OPENDAL_HDFS_ROOT=/tmp/opendal/ OPENDAL_HDFS_NAME_NODE=default OPENDAL_HDFS_ENABLE_APPEND=false diff --git a/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml b/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml index b8de8671611a..57372570ff30 100644 --- a/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml +++ b/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml @@ -21,25 +21,14 @@ description: 'Behavior test for hdfs default with atomic write dir' runs: using: "composite" steps: - - name: Setup java env - uses: actions/setup-java@v4 + - name: Setup hadoop + uses: ./.github/actions/setup-hadoop with: - distribution: temurin - java-version: "11" - - name: Setup + need-hadoop: true + - name: Setup OpenDAL env shell: bash run: | - curl -LsSf https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz | tar zxf - -C /home/runner - - export HADOOP_HOME="/home/runner/hadoop-3.3.5" - export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob) - - cp ./fixtures/hdfs/hdfs-site.xml ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml - cat << EOF >> $GITHUB_ENV - HADOOP_HOME=${HADOOP_HOME} - CLASSPATH=${CLASSPATH} - LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native OPENDAL_HDFS_ROOT=/tmp/opendal/ OPENDAL_HDFS_ATOMIC_WRITE_DIR=/tmp/atomic_write_dir/opendal/ OPENDAL_HDFS_NAME_NODE=default