From 150630ab9550332f223865c09125f3eafbd890de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ruilong=20Li=28=E6=9D=8E=E7=91=9E=E9=BE=99=29?= Date: Wed, 17 May 2023 15:20:02 -0700 Subject: [PATCH] Fix Workflows for Compiling (#224) * single build * attemp fix aws listing * fix * fix * fix * fix * fix * fix * fix * fix * fix format * fix format --- .github/workflows/building.yml | 39 ++++++++++------------------- .github/workflows/cuda/Linux-env.sh | 36 +++++++++++++------------- scripts/run_aws_listing.py | 23 +++++++---------- 3 files changed, 40 insertions(+), 58 deletions(-) diff --git a/.github/workflows/building.yml b/.github/workflows/building.yml index ee44de1b..bec9c0f4 100644 --- a/.github/workflows/building.yml +++ b/.github/workflows/building.yml @@ -121,27 +121,22 @@ jobs: if: ${{ matrix.cuda-version == 'cpu' }} run: | BUILD_NO_CUDA=1 pip install . - shell: - bash - - - name: Install main package for GPU - if: ${{ matrix.cuda-version != 'cpu' }} - run: | - source .github/workflows/cuda/${{ runner.os }}-env.sh ${{ matrix.cuda-version }} - pip install . - shell: - bash - - - name: Test installation - run: | - python -c "import nerfacc; print('nerfacc:', nerfacc.__version__)" - name: Build wheel run: | pip install wheel source .github/workflows/cuda/${{ runner.os }}-env.sh ${{ matrix.cuda-version }} python setup.py bdist_wheel --dist-dir=dist - shell: bash + shell: bash # `source` does not exist in windows powershell + + - name: Test wheel + run: | + cd dist + ls -lah + pip install *.whl + python -c "import nerfacc; print('nerfacc:', nerfacc.__version__)" + cd .. + shell: bash # `ls -lah` does not exist in windows powershell - name: Configure AWS uses: aws-actions/configure-aws-credentials@v2 @@ -169,21 +164,13 @@ jobs: - name: Upgrade pip run: | - pip install --upgrade setuptools pip install boto3 - - - name: Configure AWS - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 - name: Update AWS listing run: | python scripts/run_aws_listing.py \ - --access_key_id=${{ secrets.AWS_ACCESS_KEY_ID }} \ - --secret_access_key=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ --bucket="nerfacc-bucket" \ --region="us-west-2" - + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/cuda/Linux-env.sh b/.github/workflows/cuda/Linux-env.sh index 022aa433..543f4a37 100644 --- a/.github/workflows/cuda/Linux-env.sh +++ b/.github/workflows/cuda/Linux-env.sh @@ -4,39 +4,39 @@ case ${1} in cu118) - CUDA_HOME=/usr/local/cuda-11.8 - LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH} - PATH=${CUDA_HOME}/bin:${PATH} + export CUDA_HOME=/usr/local/cuda-11.8 + export LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH} + export PATH=${CUDA_HOME}/bin:${PATH} export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6" ;; cu117) - CUDA_HOME=/usr/local/cuda-11.7 - LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH} - PATH=${CUDA_HOME}/bin:${PATH} + export CUDA_HOME=/usr/local/cuda-11.7 + export LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH} + export PATH=${CUDA_HOME}/bin:${PATH} export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6" ;; cu116) - CUDA_HOME=/usr/local/cuda-11.6 - LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH} - PATH=${CUDA_HOME}/bin:${PATH} + export CUDA_HOME=/usr/local/cuda-11.6 + export LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH} + export PATH=${CUDA_HOME}/bin:${PATH} export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6" ;; cu115) - CUDA_HOME=/usr/local/cuda-11.5 - LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH} - PATH=${CUDA_HOME}/bin:${PATH} + export CUDA_HOME=/usr/local/cuda-11.5 + export LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH} + export PATH=${CUDA_HOME}/bin:${PATH} export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6" ;; cu113) - CUDA_HOME=/usr/local/cuda-11.3 - LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH} - PATH=${CUDA_HOME}/bin:${PATH} + export CUDA_HOME=/usr/local/cuda-11.3 + export LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH} + export PATH=${CUDA_HOME}/bin:${PATH} export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6" ;; cu102) - CUDA_HOME=/usr/local/cuda-10.2 - LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH} - PATH=${CUDA_HOME}/bin:${PATH} + export CUDA_HOME=/usr/local/cuda-10.2 + export LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH} + export PATH=${CUDA_HOME}/bin:${PATH} export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5" ;; *) diff --git a/scripts/run_aws_listing.py b/scripts/run_aws_listing.py index 6a044f05..7d11b0ea 100644 --- a/scripts/run_aws_listing.py +++ b/scripts/run_aws_listing.py @@ -1,11 +1,9 @@ import argparse from collections import defaultdict -from boto3 import client +from boto3 import resource parser = argparse.ArgumentParser() -parser.add_argument("--access_key_id", type=str, required=True) -parser.add_argument("--secret_access_key", type=str, required=True) parser.add_argument("--bucket", type=str, required=True) parser.add_argument("--region", type=str, required=True) args = parser.parse_args() @@ -19,13 +17,7 @@ "ACL": "public-read", } -s3 = client( - "s3", - aws_access_key_id=args.access_key_id, - aws_secret_access_key=args.secret_access_key, -) - -bucket = s3.Bucket(name="nerfacc-bucket") +bucket = resource("s3").Bucket(name="nerfacc-bucket") wheels_dict = defaultdict(list) for obj in bucket.objects.filter(Prefix="whl"): @@ -38,8 +30,10 @@ index_html = html.format( "\n".join( [ - href.format(f"{torch_version}.html".replace("+", "%2B"), version) - for version in wheels_dict + href.format( + f"{torch_version}.html".replace("+", "%2B"), torch_version + ) + for torch_version in wheels_dict ] ) ) @@ -53,7 +47,8 @@ "\n".join( [ href.format( - f"{ROOT_URL}/{wheel_name}".replace("+", "%2B"), wheel_name + f"{ROOT_URL}/{wheel_name}".replace("+", "%2B"), + wheel_name.split("/")[-1], ) for wheel_name in wheel_names ] @@ -63,5 +58,5 @@ with open(f"{torch_version}.html", "w") as f: f.write(torch_version_html) bucket.Object(f"whl/{torch_version}.html").upload_file( - f"{torch_version}.html", args + f"{torch_version}.html", html_args )