Fix Workflows for Compiling (#224)

* single build * attemp fix aws listing * fix * fix * fix * fix * fix * fix * fix * fix * fix format * fix format
nerfstudio-project · May 17, 2023 · 150630a · 150630a
1 parent 910488e
commit 150630a
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 58 deletions.
diff --git a/.github/workflows/building.yml b/.github/workflows/building.yml
@@ -121,27 +121,22 @@ jobs:
  if: ${{ matrix.cuda-version == 'cpu' }}
  run: |
  BUILD_NO_CUDA=1 pip install .
- shell:
- bash
-
- - name: Install main package for GPU
- if: ${{ matrix.cuda-version != 'cpu' }}
- run: |
- source .github/workflows/cuda/${{ runner.os }}-env.sh ${{ matrix.cuda-version }}
- pip install .
- shell:
- bash
-
- - name: Test installation
- run: |
- python -c "import nerfacc; print('nerfacc:', nerfacc.__version__)"
 
  - name: Build wheel
  run: |
  pip install wheel
  source .github/workflows/cuda/${{ runner.os }}-env.sh ${{ matrix.cuda-version }}
  python setup.py bdist_wheel --dist-dir=dist
- shell: bash
+ shell: bash # `source` does not exist in windows powershell
+
+ - name: Test wheel
+ run: |
+ cd dist
+ ls -lah
+ pip install *.whl
+ python -c "import nerfacc; print('nerfacc:', nerfacc.__version__)"
+ cd ..
+ shell: bash # `ls -lah` does not exist in windows powershell
 
  - name: Configure AWS
  uses: aws-actions/configure-aws-credentials@v2
@@ -169,21 +164,13 @@ jobs:
 
  - name: Upgrade pip
  run: |
- pip install --upgrade setuptools
  pip install boto3
- 
- - name: Configure AWS
- uses: aws-actions/configure-aws-credentials@v2
- with:
- aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
- aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- aws-region: us-west-2
 
  - name: Update AWS listing
  run: |
  python scripts/run_aws_listing.py \
- --access_key_id=${{ secrets.AWS_ACCESS_KEY_ID }} \
- --secret_access_key=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
  --bucket="nerfacc-bucket" \
  --region="us-west-2"
-
+ env:
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
diff --git a/.github/workflows/cuda/Linux-env.sh b/.github/workflows/cuda/Linux-env.sh
@@ -4,39 +4,39 @@
 
 case ${1} in
  cu118)
- CUDA_HOME=/usr/local/cuda-11.8
- LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
- PATH=${CUDA_HOME}/bin:${PATH}
+ export CUDA_HOME=/usr/local/cuda-11.8
+ export LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
+ export PATH=${CUDA_HOME}/bin:${PATH}
  export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"
  ;;
  cu117)
- CUDA_HOME=/usr/local/cuda-11.7
- LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
- PATH=${CUDA_HOME}/bin:${PATH}
+ export CUDA_HOME=/usr/local/cuda-11.7
+ export LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
+ export PATH=${CUDA_HOME}/bin:${PATH}
  export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"
  ;;
  cu116)
- CUDA_HOME=/usr/local/cuda-11.6
- LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
- PATH=${CUDA_HOME}/bin:${PATH}
+ export CUDA_HOME=/usr/local/cuda-11.6
+ export LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
+ export PATH=${CUDA_HOME}/bin:${PATH}
  export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"
  ;;
  cu115)
- CUDA_HOME=/usr/local/cuda-11.5
- LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
- PATH=${CUDA_HOME}/bin:${PATH}
+ export CUDA_HOME=/usr/local/cuda-11.5
+ export LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
+ export PATH=${CUDA_HOME}/bin:${PATH}
  export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"
  ;;
  cu113)
- CUDA_HOME=/usr/local/cuda-11.3
- LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
- PATH=${CUDA_HOME}/bin:${PATH}
+ export CUDA_HOME=/usr/local/cuda-11.3
+ export LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
+ export PATH=${CUDA_HOME}/bin:${PATH}
  export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5;8.0;8.6"
  ;;
  cu102)
- CUDA_HOME=/usr/local/cuda-10.2
- LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
- PATH=${CUDA_HOME}/bin:${PATH}
+ export CUDA_HOME=/usr/local/cuda-10.2
+ export LD_LIBRARY_PATH=${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}
+ export PATH=${CUDA_HOME}/bin:${PATH}
  export TORCH_CUDA_ARCH_LIST="3.5;5.0+PTX;6.0;7.0;7.5"
  ;;
  *)

diff --git a/scripts/run_aws_listing.py b/scripts/run_aws_listing.py
@@ -1,11 +1,9 @@
 import argparse
 from collections import defaultdict
 
-from boto3 import client
+from boto3 import resource
 
 parser = argparse.ArgumentParser()
-parser.add_argument("--access_key_id", type=str, required=True)
-parser.add_argument("--secret_access_key", type=str, required=True)
 parser.add_argument("--bucket", type=str, required=True)
 parser.add_argument("--region", type=str, required=True)
 args = parser.parse_args()
@@ -19,13 +17,7 @@
  "ACL": "public-read",
 }
 
-s3 = client(
- "s3",
- aws_access_key_id=args.access_key_id,
- aws_secret_access_key=args.secret_access_key,
-)
-
-bucket = s3.Bucket(name="nerfacc-bucket")
+bucket = resource("s3").Bucket(name="nerfacc-bucket")
 
 wheels_dict = defaultdict(list)
 for obj in bucket.objects.filter(Prefix="whl"):
@@ -38,8 +30,10 @@
 index_html = html.format(
  "\n".join(
  [
- href.format(f"{torch_version}.html".replace("+", "%2B"), version)
- for version in wheels_dict
+ href.format(
+ f"{torch_version}.html".replace("+", "%2B"), torch_version
+ )
+ for torch_version in wheels_dict
  ]
  )
 )
@@ -53,7 +47,8 @@
  "\n".join(
  [
  href.format(
- f"{ROOT_URL}/{wheel_name}".replace("+", "%2B"), wheel_name
+ f"{ROOT_URL}/{wheel_name}".replace("+", "%2B"),
+ wheel_name.split("/")[-1],
  )
  for wheel_name in wheel_names
  ]
@@ -63,5 +58,5 @@
  with open(f"{torch_version}.html", "w") as f:
  f.write(torch_version_html)
  bucket.Object(f"whl/{torch_version}.html").upload_file(
- f"{torch_version}.html", args
+ f"{torch_version}.html", html_args
  )