From 2a598b48b4074a806126927bec867116ab0391b0 Mon Sep 17 00:00:00 2001 From: ksimpson Date: Mon, 18 Nov 2024 11:00:17 -0800 Subject: [PATCH 01/11] just fix the build --- .github/actions/build/action.yml.j2 | 15 ++++++++++++--- .github/actions/setup/action.yml | 12 ++++++++---- .github/workflows/ci-gh.yml | 8 ++++---- .github/workflows/gh-build-and-test.yml | 4 ++++ .github/workflows/gh-build.yml | 2 +- continuous_integration/scripts/build | 6 ++++-- 6 files changed, 33 insertions(+), 14 deletions(-) diff --git a/.github/actions/build/action.yml.j2 b/.github/actions/build/action.yml.j2 index 59a6ba29..6772537d 100644 --- a/.github/actions/build/action.yml.j2 +++ b/.github/actions/build/action.yml.j2 @@ -22,6 +22,9 @@ inputs: upload-enabled: required: true type: boolean + python-version: + required: true + type: string runs: using: composite @@ -83,8 +86,14 @@ runs: sudo chown -R $(whoami) ${{ env.ARTIFACTS_DIR }} ls -lahR ${{ env.ARTIFACTS_DIR }} - - name: Upload build artifacts + - name: Upload bindings build artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ env.BINDINGS_ARTIFACT_NAME }} + path: ${{ env.BINDINGS_ARTIFACTS_DIR }} + + - name: Upload core build artifacts uses: actions/upload-artifact@v4 with: - name: ${{ env.ARTIFACT_NAME }} - path: ${{ env.ARTIFACTS_DIR }} + name: ${{ env.CORE_ARTIFACT_NAME }} + path: ${{ env.CORE_ARTIFACTS_DIR }} diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index adeb48df..2d088398 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -20,7 +20,7 @@ inputs: required: true type: boolean python-version: - required: false + required: true type: string runs: @@ -57,12 +57,16 @@ runs: PKG_DIR="${BUILD_MODE}/${TARGET_PLATFORM}" fi - echo "ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ inputs.client-repo }}-python${{ inputs.python-version }}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV - echo "ARTIFACTS_DIR=$(realpath "$(pwd)/dist")" >> $GITHUB_ENV + PYTHON_VERSION_FORMATTED=$(echo '${{ inputs.python-version }}' | tr -d '.') + + echo "BINDINGS_ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-cuda_bindings-python${PYTHON_VERSION_FORMATTED}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV + echo "BINDINGS_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_bindings/dist")" >> $GITHUB_ENV + echo "CORE_ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-cuda_core-python${PYTHON_VERSION_FORMATTED}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV + echo "CORE_ARTIFACTS_DIR=$(realpath "$(pwd)/cuda_core/dist")" >> $GITHUB_ENV echo "USE_CUDA=${{ (inputs.target-device == 'cpu' && 'OFF') || 'ON' }}" >> $GITHUB_ENV echo "UPLOAD_ENABLED=${{ (inputs.upload-enabled == 'true' && 'ON') || 'OFF' }}" >> $GITHUB_ENV echo "LEGATE_CORE_BUILD_MODE=${BUILD_MODE}" >> $GITHUB_ENV echo "BUILD_DATE=$(date +%Y%m%d)" >> $GITHUB_ENV echo "TARGET_PLATFORM=${TARGET_PLATFORM}" >> $GITHUB_ENV echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV - echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV \ No newline at end of file + echo "PYTHON_VERSION=${{inputs.python-version}}" >> $GITHUB_ENV \ No newline at end of file diff --git a/.github/workflows/ci-gh.yml b/.github/workflows/ci-gh.yml index 2c43d03f..cff86a40 100644 --- a/.github/workflows/ci-gh.yml +++ b/.github/workflows/ci-gh.yml @@ -5,10 +5,7 @@ concurrency: cancel-in-progress: true on: - push: - branches: - - "pull-request/[0-9]+" - - "main" + [push] jobs: build-and-test: @@ -24,6 +21,8 @@ jobs: - release upload-enabled: - false + python-version: + - 3.12 uses: ./.github/workflows/gh-build-and-test.yml with: @@ -32,4 +31,5 @@ jobs: build-mode: ${{ matrix.build-mode }} build-type: ci upload-enabled: ${{ matrix.upload-enabled }} + python-version: ${{ matrix.python-version }} secrets: inherit diff --git a/.github/workflows/gh-build-and-test.yml b/.github/workflows/gh-build-and-test.yml index 4376776d..430fbf5b 100644 --- a/.github/workflows/gh-build-and-test.yml +++ b/.github/workflows/gh-build-and-test.yml @@ -16,6 +16,9 @@ on: upload-enabled: type: boolean required: true + python-version: + type: string + required: true jobs: build: if: ${{ github.repository_owner == 'nvidia' }} @@ -31,4 +34,5 @@ jobs: dependencies-file: "" build-mode: ${{ inputs.build-mode }} upload-enabled: ${{ inputs.upload-enabled }} + python-version: ${{ inputs.python-version }} secrets: inherit diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 922e04ec..27754b0d 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -33,7 +33,7 @@ on: required: true type: boolean python-version: - required: false + required: true type: string jobs: diff --git a/continuous_integration/scripts/build b/continuous_integration/scripts/build index 5db25e67..618edd5f 100755 --- a/continuous_integration/scripts/build +++ b/continuous_integration/scripts/build @@ -3,11 +3,13 @@ build_ci() { set -xeou pipefail - cd "${REPO_DIR}" - export CUDA_HOME="${CONDA_PREFIX}/targets/x86_64-linux" export PARALLEL_LEVEL=$(nproc --ignore 1) + cd "${REPO_DIR}/cuda_bindings" + python setup.py bdist_wheel + + cd "${REPO_DIR}/cuda_core" python setup.py bdist_wheel } From 8c02036013ab908a2cb56f063c9e71d47d1835cb Mon Sep 17 00:00:00 2001 From: ksimpson Date: Mon, 18 Nov 2024 11:08:49 -0800 Subject: [PATCH 02/11] print some info --- .github/actions/build/action.yml.j2 | 32 ++++++++++++++----- continuous_integration/scripts/make-conda-env | 11 ------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/.github/actions/build/action.yml.j2 b/.github/actions/build/action.yml.j2 index 6772537d..a4b41817 100644 --- a/.github/actions/build/action.yml.j2 +++ b/.github/actions/build/action.yml.j2 @@ -31,6 +31,14 @@ runs: steps: <% for package_id, package_info in packages.items() %> + - name: Print package info for <> + run: | + echo "Package ID: <>" + echo "Repository: <>" + echo "Artifact Name: <>" + echo "Git Tag: <>" + echo "Artifact Workflow: <>" + - name: Download <> (artifacts) uses: ./.github/actions/download-artifacts with: @@ -39,7 +47,7 @@ runs: target-device: "${{ inputs.target-device }}" git_sha: "<>" host-platform: ${{ inputs.host-platform }} - dest-dir: ${{ env.ARTIFACTS_DIR }} + dest-dir: "${{ env.REPO_DIR }}/<>" dependencies-workflow: <> <% endfor %> @@ -63,14 +71,16 @@ runs: -e AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY \ -e GITHUB_TOKEN \ - -e ARTIFACTS_DIR="$ARTIFACTS_DIR" \ + -e BINDINGS_ARTIFACTS_DIR="$BINDINGS_ARTIFACTS_DIR" \ + -e CORE_ARTIFACTS_DIR="$CORE_ARTIFACTS_DIR" \ -e UPLOAD_ENABLED="$UPLOAD_ENABLED" \ -e USE_CUDA="$USE_CUDA" \ -e REPO_DIR="$REPO_DIR" \ -e LEGATE_CORE_BUILD_MODE="$LEGATE_CORE_BUILD_MODE" \ -e PYTHON_VERSION="$PYTHON_VERSION" \ -v "${{ env.REPO_DIR }}:${{ env.REPO_DIR }}" \ - -v "${{ env.ARTIFACTS_DIR }}:${{ env.ARTIFACTS_DIR }}" \ + -v "${{ env.BINDINGS_ARTIFACTS_DIR }}:${{ env.BINDINGS_ARTIFACTS_DIR }}" \ + -v "${{ env.CORE_ARTIFACTS_DIR }}:${{ env.CORE_ARTIFACTS_DIR }}" \ --rm "${{ inputs.docker-image }}" \ /bin/bash -c "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint ${{ env.REPO_DIR }}/continuous_integration/scripts/build ${{ inputs.build-type}} ${{ inputs.target-device }}" @@ -80,20 +90,26 @@ runs: run: | "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint" "${{ env.REPO_DIR }}/continuous_integration/scripts/build" "${{ inputs.build-type}}" "${{ inputs.target-device }}" - - name: Display structure of the artifacts folder (post build) + - name: Display structure of the bindings artifacts folder (post build) shell: bash --noprofile --norc -xeuo pipefail {0} run: | - sudo chown -R $(whoami) ${{ env.ARTIFACTS_DIR }} - ls -lahR ${{ env.ARTIFACTS_DIR }} + sudo chown -R $(whoami) ${{ env.BINDINGS_ARTIFACTS_DIR }} + ls -lahR ${{ env.BINDINGS_ARTIFACTS_DIR }} - name: Upload bindings build artifacts uses: actions/upload-artifact@v4 with: name: ${{ env.BINDINGS_ARTIFACT_NAME }} - path: ${{ env.BINDINGS_ARTIFACTS_DIR }} + path: ${{ env.BINDINGS_ARTIFACTS_DIR }}\ + + - name: Display structure of the core artifacts folder (post build) + shell: bash --noprofile --norc -xeuo pipefail {0} + run: | + sudo chown -R $(whoami) ${{ env.CORE_ARTIFACTS_DIR }} + ls -lahR ${{ env.CORE_ARTIFACTS_DIR }} - name: Upload core build artifacts uses: actions/upload-artifact@v4 with: name: ${{ env.CORE_ARTIFACT_NAME }} - path: ${{ env.CORE_ARTIFACTS_DIR }} + path: ${{ env.CORE_ARTIFACTS_DIR }}\ diff --git a/continuous_integration/scripts/make-conda-env b/continuous_integration/scripts/make-conda-env index 1294f038..d2b86f28 100755 --- a/continuous_integration/scripts/make-conda-env +++ b/continuous_integration/scripts/make-conda-env @@ -6,17 +6,6 @@ make_ci_env() { mamba env create -n "${CONDA_ENV}" -f "${REPO_DIR}/continuous_integration/environment.yml" } -make_test_env() { - . conda-utils - - mamba env create -n "${CONDA_ENV}" -f "${REPO_DIR}/continuous_integration/environment.yml" - - activate_conda_env - - pip install "${ARTIFACTS_DIR}"/*.whl - -} - make_conda_env() { set -xeuo pipefail From e187df0c41379c75a90dbd06e25b9c7da8698baa Mon Sep 17 00:00:00 2001 From: ksimpson Date: Mon, 18 Nov 2024 11:15:44 -0800 Subject: [PATCH 03/11] print some info --- .github/actions/build/action.yml.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/build/action.yml.j2 b/.github/actions/build/action.yml.j2 index a4b41817..b548a12b 100644 --- a/.github/actions/build/action.yml.j2 +++ b/.github/actions/build/action.yml.j2 @@ -100,7 +100,7 @@ runs: uses: actions/upload-artifact@v4 with: name: ${{ env.BINDINGS_ARTIFACT_NAME }} - path: ${{ env.BINDINGS_ARTIFACTS_DIR }}\ + path: ${{ env.BINDINGS_ARTIFACTS_DIR }} - name: Display structure of the core artifacts folder (post build) shell: bash --noprofile --norc -xeuo pipefail {0} @@ -112,4 +112,4 @@ runs: uses: actions/upload-artifact@v4 with: name: ${{ env.CORE_ARTIFACT_NAME }} - path: ${{ env.CORE_ARTIFACTS_DIR }}\ + path: ${{ env.CORE_ARTIFACTS_DIR }} From 306e933dc442ed3292abf75a749486d220e850b5 Mon Sep 17 00:00:00 2001 From: ksimpson Date: Mon, 18 Nov 2024 11:24:31 -0800 Subject: [PATCH 04/11] build both bindings and core, and upload them appropriately --- .github/actions/build/action.yml.j2 | 8 -------- .github/workflows/ci-gh.yml | 3 ++- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/actions/build/action.yml.j2 b/.github/actions/build/action.yml.j2 index b548a12b..e74ee36c 100644 --- a/.github/actions/build/action.yml.j2 +++ b/.github/actions/build/action.yml.j2 @@ -31,14 +31,6 @@ runs: steps: <% for package_id, package_info in packages.items() %> - - name: Print package info for <> - run: | - echo "Package ID: <>" - echo "Repository: <>" - echo "Artifact Name: <>" - echo "Git Tag: <>" - echo "Artifact Workflow: <>" - - name: Download <> (artifacts) uses: ./.github/actions/download-artifacts with: diff --git a/.github/workflows/ci-gh.yml b/.github/workflows/ci-gh.yml index cff86a40..7e2f06c4 100644 --- a/.github/workflows/ci-gh.yml +++ b/.github/workflows/ci-gh.yml @@ -5,7 +5,7 @@ concurrency: cancel-in-progress: true on: - [push] + [push, pull_request, workflow_dispatch, workflow_call] jobs: build-and-test: @@ -22,6 +22,7 @@ jobs: upload-enabled: - false python-version: + #TODO cover the whole python and cuda matrix - 3.12 uses: ./.github/workflows/gh-build-and-test.yml From f03d7ea83096444d2e453f824a8ce828638a3e9d Mon Sep 17 00:00:00 2001 From: ksimpson Date: Mon, 18 Nov 2024 11:32:06 -0800 Subject: [PATCH 05/11] build both bindings and core, and upload them appropriately --- .github/workflows/ci-gh.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-gh.yml b/.github/workflows/ci-gh.yml index 7e2f06c4..d38cb8e3 100644 --- a/.github/workflows/ci-gh.yml +++ b/.github/workflows/ci-gh.yml @@ -5,7 +5,10 @@ concurrency: cancel-in-progress: true on: - [push, pull_request, workflow_dispatch, workflow_call] + push: + branches: + - "pull-request/[0-9]+" + - "main" jobs: build-and-test: From 13f8b9ce63a284d5d942416fef52f909a15b103d Mon Sep 17 00:00:00 2001 From: sandeepd-nv Date: Tue, 19 Nov 2024 21:41:02 +0530 Subject: [PATCH 06/11] Remove unused code. --- .../build/{action.yml.j2 => action.yml} | 24 +------- .github/actions/download-artifacts/action.yml | 59 ------------------- .github/workflows/gh-build.yml | 21 ------- continuous_integration/no_dependencies.json | 1 - .../scripts/render-template.py | 52 ---------------- 5 files changed, 1 insertion(+), 156 deletions(-) rename .github/actions/build/{action.yml.j2 => action.yml} (76%) delete mode 100644 .github/actions/download-artifacts/action.yml delete mode 100644 continuous_integration/no_dependencies.json delete mode 100755 continuous_integration/scripts/render-template.py diff --git a/.github/actions/build/action.yml.j2 b/.github/actions/build/action.yml similarity index 76% rename from .github/actions/build/action.yml.j2 rename to .github/actions/build/action.yml index e74ee36c..952fb9cd 100644 --- a/.github/actions/build/action.yml.j2 +++ b/.github/actions/build/action.yml @@ -30,28 +30,6 @@ runs: using: composite steps: -<% for package_id, package_info in packages.items() %> - - name: Download <> (artifacts) - uses: ./.github/actions/download-artifacts - with: - artifact-repo: "<>" - artifact-name: "<>" - target-device: "${{ inputs.target-device }}" - git_sha: "<>" - host-platform: ${{ inputs.host-platform }} - dest-dir: "${{ env.REPO_DIR }}/<>" - dependencies-workflow: <> -<% endfor %> - -<% if packages %> - - - name: Display structure of downloaded artifacts - shell: bash --noprofile --norc -xeuo pipefail {0} - run: | - pwd - ls -lahR ${{ env.ARTIFACTS_DIR }} -<% endif %> - - if: ${{ inputs.use-container }} name: Build (in container) shell: bash --noprofile --norc -xeuo pipefail {0} @@ -93,7 +71,7 @@ runs: with: name: ${{ env.BINDINGS_ARTIFACT_NAME }} path: ${{ env.BINDINGS_ARTIFACTS_DIR }} - + - name: Display structure of the core artifacts folder (post build) shell: bash --noprofile --norc -xeuo pipefail {0} run: | diff --git a/.github/actions/download-artifacts/action.yml b/.github/actions/download-artifacts/action.yml deleted file mode 100644 index c3dffa02..00000000 --- a/.github/actions/download-artifacts/action.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: download-artifacts - -description: Download dependencies (artifacts) - -inputs: - artifact-repo: - type: string - require: true - artifact-name: - type: string - require: true - target-device: - type: string - required: true - git_sha: - type: string - required: true - host-platform: - type: string - required: true - dest-dir: - type: string - required: true - dependencies-workflow: - required: true - type: string - -runs: - using: composite - steps: - - - id: cache - name: Cache conda artifacts - uses: actions/cache@v4 - with: - key: "nvidia/{ inputs.artifact-repo }}@${{ inputs.host-platform }}-${{ inputs.git_sha }}-${{ inputs.target-device }}" - path: ${{ inputs.dest-dir }} - - - if: steps.cache.outputs.cache-hit != 'true' - name: Download ${{ inputs.artifact-repo }} artifacts - uses: dawidd6/action-download-artifact@v3 - with: - path: ${{ inputs.dest-dir }} - repo: nvidia/${{ inputs.artifact-repo }} - check_artifacts: true - commit: ${{ inputs.git_sha }} - workflow_conclusion: "" - workflow: ${{ inputs.dependencies-workflow }} - name: ${{ inputs.artifact-name }} - skip_unpack: true - if_no_artifact_found: fail - allow_forks: false - - - if: steps.cache.outputs.cache-hit != 'true' - name: Unpack artifact - shell: bash --noprofile --norc -xeuo pipefail {0} - run: | - cd ${{ inputs.dest-dir }} - unzip *.zip diff --git a/.github/workflows/gh-build.yml b/.github/workflows/gh-build.yml index 27754b0d..c60e0c2a 100644 --- a/.github/workflows/gh-build.yml +++ b/.github/workflows/gh-build.yml @@ -63,27 +63,6 @@ jobs: upload-enabled: ${{ inputs.upload-enabled }} python-version: ${{ inputs.python-version }} - - name: Render templates - shell: bash --noprofile --norc -xeuo pipefail {0} - run: | - pip -q install jinja2 - - DEPENDENCIES_FILE="" - - if [ -z "${{ inputs.dependencies-file }}" ]; then - DEPENDENCIES_FILE="${REPO_DIR}/continuous_integration/no_dependencies.json" - else - DEPENDENCIES_FILE="${REPO_DIR}/${{ inputs.dependencies-file }}" - fi - - ${REPO_DIR}/continuous_integration/scripts/render-template.py .github/actions/build/action.yml.j2 "${DEPENDENCIES_FILE}" .github/actions/build/action.yml - - - name: Dump templates - shell: bash --noprofile --norc -xeuo pipefail {0} - run: | - echo ${REPO_DIR}/.github/actions/build/action.yml - cat ${REPO_DIR}/.github/actions/build/action.yml - - name: Call build action uses: ./.github/actions/build with: diff --git a/continuous_integration/no_dependencies.json b/continuous_integration/no_dependencies.json deleted file mode 100644 index e2d7bd79..00000000 --- a/continuous_integration/no_dependencies.json +++ /dev/null @@ -1 +0,0 @@ -{ "packages" : {} } diff --git a/continuous_integration/scripts/render-template.py b/continuous_integration/scripts/render-template.py deleted file mode 100755 index b887e361..00000000 --- a/continuous_integration/scripts/render-template.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import json -from jinja2 import Environment, FileSystemLoader -import os -import re - -# TODO: make this work for arbitrary context. ie. implement replace_using_context() -def replace_placeholder(source_str, variable_name, variable_value): - # Escaping any regex special characters in variable_name - variable_name_escaped = re.escape(variable_name) - - # Using regular expression to replace ${variable_name} with actual variable_value - # \s* means any amount of whitespace (including none) - # pattern = rf'\$\{{\s*\{{\s*{variable_name_escaped}\s*\}}\s*\}}' - pattern = rf'<<\s*{variable_name_escaped}\s*>>' - return re.sub(pattern, variable_value.strip(), source_str) - -# Setup command-line argument parsing -parser = argparse.ArgumentParser(description='Render a Jinja2 template using a JSON context.') -parser.add_argument('template_file', type=str, help='Path to the Jinja2 template file (with .j2 extension).') -parser.add_argument('json_file', type=str, help='Path to the JSON file to use as the rendering context.') -parser.add_argument('output_file', type=str, help='Path to the output file.') - -args = parser.parse_args() - -# Load JSON file as the rendering context -with open(args.json_file, 'r') as file: - context = json.load(file) - -# Setup Jinja2 environment and load the template -env = Environment( - loader=FileSystemLoader(searchpath='./'), - variable_start_string='<<', - variable_end_string='>>', - block_start_string='<%', - block_end_string='%>', - comment_start_string='<#', - comment_end_string='#>') -env.filters['replace_placeholder'] = replace_placeholder - -template = env.get_template(args.template_file) - -# Render the template with the context -rendered_content = template.render(context) -# print(rendered_content) - -with open(args.output_file, 'w') as file: - file.write(rendered_content) - -print(f'Template rendered successfully. Output saved to {args.output_file}') From a90ac5b6533cdb2bc79f7dd4aca3157e817440de Mon Sep 17 00:00:00 2001 From: sandeepd-nv Date: Tue, 19 Nov 2024 21:49:45 +0530 Subject: [PATCH 07/11] Minor change. --- .github/actions/setup/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 2d088398..c2a8407c 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -69,4 +69,4 @@ runs: echo "BUILD_DATE=$(date +%Y%m%d)" >> $GITHUB_ENV echo "TARGET_PLATFORM=${TARGET_PLATFORM}" >> $GITHUB_ENV echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV - echo "PYTHON_VERSION=${{inputs.python-version}}" >> $GITHUB_ENV \ No newline at end of file + echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV From 3cc3832b7cef18d73406f54c0c32475c2968519e Mon Sep 17 00:00:00 2001 From: ksimpson Date: Wed, 20 Nov 2024 09:42:45 -0800 Subject: [PATCH 08/11] wire the python version to env creation --- continuous_integration/scripts/make-conda-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/scripts/make-conda-env b/continuous_integration/scripts/make-conda-env index d2b86f28..6e30ffb4 100755 --- a/continuous_integration/scripts/make-conda-env +++ b/continuous_integration/scripts/make-conda-env @@ -3,7 +3,7 @@ set -x make_ci_env() { - mamba env create -n "${CONDA_ENV}" -f "${REPO_DIR}/continuous_integration/environment.yml" + mamba env create -n "${CONDA_ENV}" python="${PYTHON_VERSION} -f "${REPO_DIR}/continuous_integration/environment.yml" } make_conda_env() { From a6ddd1ad281eaf0e3474ce66639d7cef941bc3e5 Mon Sep 17 00:00:00 2001 From: ksimpson Date: Wed, 20 Nov 2024 10:38:37 -0800 Subject: [PATCH 09/11] fix typo --- continuous_integration/scripts/make-conda-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/scripts/make-conda-env b/continuous_integration/scripts/make-conda-env index 6e30ffb4..53cfe227 100755 --- a/continuous_integration/scripts/make-conda-env +++ b/continuous_integration/scripts/make-conda-env @@ -3,7 +3,7 @@ set -x make_ci_env() { - mamba env create -n "${CONDA_ENV}" python="${PYTHON_VERSION} -f "${REPO_DIR}/continuous_integration/environment.yml" + mamba env create -n "${CONDA_ENV}" python="${PYTHON_VERSION}" -f "${REPO_DIR}/continuous_integration/environment.yml" } make_conda_env() { From 404e9d4d0441e60b276dc81e96f46136ca638436 Mon Sep 17 00:00:00 2001 From: ksimpson Date: Wed, 20 Nov 2024 16:22:52 -0800 Subject: [PATCH 10/11] make the python and cuda version an argument to an autogenerator for environment.yml --- .../scripts/generate-environment | 36 +++++++++++++++++++ continuous_integration/scripts/make-conda-env | 5 ++- 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100755 continuous_integration/scripts/generate-environment diff --git a/continuous_integration/scripts/generate-environment b/continuous_integration/scripts/generate-environment new file mode 100755 index 00000000..3834b41f --- /dev/null +++ b/continuous_integration/scripts/generate-environment @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# Function to generate environment.yml +generate_environment_yml() { + local python_version=$1 + local cuda_version=$2 + local output_path=$3 + + cat < "${output_path}/environment.yml" +name: cuda_python +channels: + - defaults + - conda-forge +dependencies: + - python=${python_version} + - cython + - pytest + - numpy + - setuptools + - wheel + - pip + - cuda-version=${cuda_version} + - cuda-cudart-static + - cuda-driver-dev + - cuda-cudart-dev + - cuda-profiler-api + - cuda-nvrtc-dev + - cuda-nvcc + - pip: + - pytest-benchmark + - pyclibrary + - versioneer==0.29 + - tomli; python_version < "3.11" + - pywin32; sys_platform == 'win32' +EOF +} \ No newline at end of file diff --git a/continuous_integration/scripts/make-conda-env b/continuous_integration/scripts/make-conda-env index 53cfe227..37539b37 100755 --- a/continuous_integration/scripts/make-conda-env +++ b/continuous_integration/scripts/make-conda-env @@ -3,13 +3,16 @@ set -x make_ci_env() { - mamba env create -n "${CONDA_ENV}" python="${PYTHON_VERSION}" -f "${REPO_DIR}/continuous_integration/environment.yml" + #TODO wire cuda version as a top level matrix argument + generate_environment_yml "${PYTHON_VERSION}" 12.6 . + mamba env create -n "${CONDA_ENV}" -f ./environment.yml } make_conda_env() { set -xeuo pipefail . setup-utils; + . generate-environment set_base_defs; case "$1" in From 3a6f68e9ec4a9606220484a0cbf9d8fe83161688 Mon Sep 17 00:00:00 2001 From: ksimpson Date: Mon, 25 Nov 2024 09:43:52 -0800 Subject: [PATCH 11/11] fix line ending --- .../scripts/generate-environment | 70 +++++++++---------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/continuous_integration/scripts/generate-environment b/continuous_integration/scripts/generate-environment index 3834b41f..8bf2c38d 100755 --- a/continuous_integration/scripts/generate-environment +++ b/continuous_integration/scripts/generate-environment @@ -1,36 +1,36 @@ -#!/usr/bin/env bash - -# Function to generate environment.yml -generate_environment_yml() { - local python_version=$1 - local cuda_version=$2 - local output_path=$3 - - cat < "${output_path}/environment.yml" -name: cuda_python -channels: - - defaults - - conda-forge -dependencies: - - python=${python_version} - - cython - - pytest - - numpy - - setuptools - - wheel - - pip - - cuda-version=${cuda_version} - - cuda-cudart-static - - cuda-driver-dev - - cuda-cudart-dev - - cuda-profiler-api - - cuda-nvrtc-dev - - cuda-nvcc - - pip: - - pytest-benchmark - - pyclibrary - - versioneer==0.29 - - tomli; python_version < "3.11" - - pywin32; sys_platform == 'win32' -EOF +#!/usr/bin/env bash + +# Function to generate environment.yml +generate_environment_yml() { + local python_version=$1 + local cuda_version=$2 + local output_path=$3 + + cat < "${output_path}/environment.yml" +name: cuda_python +channels: + - defaults + - conda-forge +dependencies: + - python=${python_version} + - cython + - pytest + - numpy + - setuptools + - wheel + - pip + - cuda-version=${cuda_version} + - cuda-cudart-static + - cuda-driver-dev + - cuda-cudart-dev + - cuda-profiler-api + - cuda-nvrtc-dev + - cuda-nvcc + - pip: + - pytest-benchmark + - pyclibrary + - versioneer==0.29 + - tomli; python_version < "3.11" + - pywin32; sys_platform == 'win32' +EOF } \ No newline at end of file