[Validations] Add option to use meta cdn for pypi index (#1999)

* [Validations] Add option to use meta cdn for pypi index * fix * test
pytorch · Sep 27, 2024 · 536fefe · 536fefe
1 parent 89d9a8c
commit 536fefe
Show file tree

Hide file tree

Showing 6 changed files with 70 additions and 60 deletions.
diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh
@@ -17,9 +17,9 @@ else
  # Please note ffmpeg is required for torchaudio, see https://github.com/pytorch/pytorch/issues/96159
  conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy ffmpeg
  conda activate ${ENV_NAME}
- 
+
  # Remove when https://github.com/pytorch/builder/issues/1985 is fixed
- if [[ ${MATRIX_GPU_ARCH_TYPE} == 'cuda-aarch64' ]]; then 
+ if [[ ${MATRIX_GPU_ARCH_TYPE} == 'cuda-aarch64' ]]; then
  pip3 install numpy --force-reinstall
  fi
 
@@ -35,6 +35,12 @@ else
  INSTALLATION=${INSTALLATION/"--index-url"/"--extra-index-url"}
  fi
 
+ # use-meta-cdn: use meta cdn for pypi download
+ if [[ ${USE_META_CDN} == 'true' ]]; then
+ INSTALLATION=${INSTALLATION/"download.pytorch.org"/"d3kup0pazkvub8.cloudfront.net"}
+ fi
+
+
  if [[ ${TORCH_ONLY} == 'true' ]]; then
  INSTALLATION=${INSTALLATION/"torchvision torchaudio"/""}
  TEST_SUFFIX=" --package torchonly"

diff --git a/.github/workflows/validate-aarch64-linux-binaries.yml b/.github/workflows/validate-aarch64-linux-binaries.yml
@@ -7,11 +7,6 @@ on:
  description: "Channel to use (nightly, test, release, all)"
  required: true
  type: string
- ref:
- description: 'Reference to checkout, defaults to empty'
- default: ""
- required: false
- type: string
  torchonly:
  description: 'Validate torchonly'
  default: false
@@ -42,6 +37,11 @@ on:
  default: false
  required: false
  type: boolean
+ use-meta-cdn:
+ description: 'Use meta cdn for installing pip binaries'
+ default: false
+ required: false
+ type: boolean
  use-extra-index-url:
  description: 'Use extra-index url for pip tests'
  default: false
@@ -58,11 +58,6 @@ on:
  - nightly
  - test
  - all
- ref:
- description: 'Reference to checkout, defaults to empty'
- default: ""
- required: false
- type: string
  torchonly:
  description: 'Validate torchonly'
  default: false
@@ -88,6 +83,11 @@ on:
  default: false
  required: false
  type: boolean
+ use-meta-cdn:
+ description: 'Use meta cdn for installing pip binaries'
+ default: false
+ required: false
+ type: boolean
  use-extra-index-url:
  description: 'Use extra-index url for pip tests'
  default: false
@@ -114,7 +114,7 @@ jobs:
  with:
  runner: ${{ matrix.validation_runner }}
  repository: "pytorch/builder"
- ref: ${{ inputs.ref || github.ref }}
+ ref: ${{ github.ref }}
  job-name: ${{ matrix.build_name }}
  docker-image: ${{ matrix.container_image }}
  binary-matrix: ${{ toJSON(matrix) }}
@@ -131,6 +131,7 @@ jobs:
  export RELEASE_VERSION=${{ inputs.version }}
  export USE_FORCE_REINSTALL=${{ inputs.use-force-reinstall }}
  export USE_EXTRA_INDEX_URL=${{ inputs.use-extra-index-url }}
+ export USE_META_CDN=${{ inputs.use-meta-cdn }}
  export USE_VERSION_SET=${{ inputs.use-version-set }}
  if [[ ${USE_VERSION_SET} == 'true' ]]; then
  export VISION_RELEASE_VERSION=${{ fromJson(inputs.release-matrix).torchvision }}

diff --git a/.github/workflows/validate-binaries.yml b/.github/workflows/validate-binaries.yml
@@ -17,11 +17,6 @@ on:
  description: "Channel to use (nightly, test, release, all)"
  required: true
  type: string
- ref:
- description: 'Reference to checkout, defaults to empty'
- default: ""
- required: false
- type: string
  torchonly:
  description: 'Validate torchonly'
  default: false
@@ -47,6 +42,11 @@ on:
  default: false
  required: false
  type: boolean
+ use-meta-cdn:
+ description: 'Use meta cdn for installing pip binaries'
+ default: false
+ required: false
+ type: boolean
  use_split_build:
  description: |
  [Experimental] Use Split Build
@@ -81,11 +81,6 @@ on:
  - nightly
  - test
  - all
- ref:
- description: 'Reference to checkout, defaults to empty'
- default: ""
- required: false
- type: string
  torchonly:
  description: 'Validate torchonly'
  default: false
@@ -111,6 +106,11 @@ on:
  default: false
  required: false
  type: boolean
+ use-meta-cdn:
+ description: 'Use meta cdn for installing pip binaries'
+ default: false
+ required: false
+ type: boolean
  use_split_build:
  description: |
  [Experimental] Use Split Build
@@ -136,21 +136,20 @@ jobs:
  uses: ./.github/workflows/validate-windows-binaries.yml
  with:
  channel: ${{ inputs.channel }}
- ref: ${{ inputs.ref || github.ref }}
  torchonly: ${{ inputs.torchonly }}
  version: ${{ inputs.version }}
  release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
  use-only-dl-pytorch-org: ${{ inputs.use-only-dl-pytorch-org }}
  use-force-reinstall: ${{ inputs.use-force-reinstall }}
  use-extra-index-url: ${{ inputs.use-extra-index-url }}
+ use-meta-cdn: ${{ inputs.use-meta-cdn }}
 
  linux:
  if: inputs.os == 'linux' || inputs.os == 'all'
  needs: generate-release-matrix
  uses: ./.github/workflows/validate-linux-binaries.yml
  with:
  channel: ${{ inputs.channel }}
- ref: ${{ inputs.ref || github.ref }}
  torchonly: ${{ inputs.torchonly }}
  version: ${{ inputs.version }}
  release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
@@ -159,31 +158,32 @@ jobs:
  use-force-reinstall: ${{ inputs.use-force-reinstall }}
  use_split_build: ${{ inputs.use_split_build }}
  use-extra-index-url: ${{ inputs.use-extra-index-url }}
+ use-meta-cdn: ${{ inputs.use-meta-cdn }}
 
  linux-aarch64:
  if: inputs.os == 'linux-aarch64' || inputs.os == 'all'
  needs: generate-release-matrix
  uses: ./.github/workflows/validate-aarch64-linux-binaries.yml
  with:
  channel: ${{ inputs.channel }}
- ref: ${{ inputs.ref || github.ref }}
  torchonly: ${{ inputs.torchonly }}
  version: ${{ inputs.version }}
  release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
  use-only-dl-pytorch-org: ${{ inputs.use-only-dl-pytorch-org }}
  use-force-reinstall: ${{ inputs.use-force-reinstall }}
  use-extra-index-url: ${{ inputs.use-extra-index-url }}
+ use-meta-cdn: ${{ inputs.use-meta-cdn }}
 
  mac-arm64:
  if: inputs.os == 'macos' || inputs.os == 'all'
  needs: generate-release-matrix
  uses: ./.github/workflows/validate-macos-arm64-binaries.yml
  with:
  channel: ${{ inputs.channel }}
- ref: ${{ inputs.ref || github.ref }}
  torchonly: ${{ inputs.torchonly }}
  version: ${{ inputs.version }}
  release-matrix: ${{ needs.generate-release-matrix.outputs.matrix }}
  use-only-dl-pytorch-org: ${{ inputs.use-only-dl-pytorch-org }}
  use-force-reinstall: ${{ inputs.use-force-reinstall }}
  use-extra-index-url: ${{ inputs.use-extra-index-url }}
+ use-meta-cdn: ${{ inputs.use-meta-cdn }}
diff --git a/.github/workflows/validate-linux-binaries.yml b/.github/workflows/validate-linux-binaries.yml
@@ -7,11 +7,6 @@ on:
  description: "Channel to use (nightly, test, release, all)"
  required: true
  type: string
- ref:
- description: 'Reference to checkout, defaults to empty'
- default: ""
- required: false
- type: string
  torchonly:
  description: 'Validate torchonly'
  default: false
@@ -47,6 +42,11 @@ on:
  default: false
  required: false
  type: boolean
+ use-meta-cdn:
+ description: 'Use meta cdn for installing pip binaries'
+ default: false
+ required: false
+ type: boolean
  use_split_build:
  description: |
  [Experimental] Use split build
@@ -69,11 +69,6 @@ on:
  - nightly
  - test
  - all
- ref:
- description: 'Reference to checkout, defaults to empty'
- default: ""
- required: false
- type: string
  torchonly:
  description: 'Validate torchonly'
  default: false
@@ -104,6 +99,11 @@ on:
  default: false
  required: false
  type: boolean
+ use-meta-cdn:
+ description: 'Use meta cdn for installing pip binaries'
+ default: false
+ required: false
+ type: boolean
  use_split_build:
  description: |
  [Experimental] Use split build
@@ -137,7 +137,7 @@ jobs:
  with:
  runner: ${{ matrix.validation_runner }}
  repository: "pytorch/builder"
- ref: ${{ inputs.ref || github.ref }}
+ ref: ${{ github.ref }}
  job-name: ${{ matrix.build_name }}
  docker-image: ${{ matrix.gpu_arch_type == 'xpu' && matrix.container_image || 'pytorch/conda-builder' }}
  binary-matrix: ${{ toJSON(matrix) }}
@@ -149,6 +149,7 @@ jobs:
  export INCLUDE_TEST_OPS=${{ inputs.include-test-ops }}
  export USE_ONLY_DL_PYTORCH_ORG=${{ inputs.use-only-dl-pytorch-org }}
  export USE_EXTRA_INDEX_URL=${{ inputs.use-extra-index-url }}
+ export USE_META_CDN=${{ inputs.use-meta-cdn }}
  export RELEASE_VERSION=${{ inputs.version }}
  export USE_VERSION_SET=${{ inputs.use-version-set }}
  if [[ ${USE_VERSION_SET} == 'true' ]]; then

diff --git a/.github/workflows/validate-macos-arm64-binaries.yml b/.github/workflows/validate-macos-arm64-binaries.yml
@@ -7,11 +7,6 @@ on:
  description: "Channel to use (nightly, test, release, all)"
  required: true
  type: string
- ref:
- description: 'Reference to checkout, defaults to empty'
- default: ""
- required: false
- type: string
  torchonly:
  description: 'Validate torchonly'
  default: false
@@ -42,6 +37,11 @@ on:
  default: false
  required: false
  type: boolean
+ use-meta-cdn:
+ description: 'Use meta cdn for installing pip binaries'
+ default: false
+ required: false
+ type: boolean
  use-extra-index-url:
  description: 'Use extra-index url for pip tests'
  default: false
@@ -58,11 +58,6 @@ on:
  - nightly
  - test
  - all
- ref:
- description: 'Reference to checkout, defaults to empty'
- default: ""
- required: false
- type: string
  torchonly:
  description: 'Validate torchonly'
  default: false
@@ -88,6 +83,11 @@ on:
  default: false
  required: false
  type: boolean
+ use-meta-cdn:
+ description: 'Use meta cdn for installing pip binaries'
+ default: false
+ required: false
+ type: boolean
  use-extra-index-url:
  description: 'Use extra-index url for pip tests'
  default: false
@@ -113,7 +113,7 @@ jobs:
  with:
  runner: ${{ matrix.validation_runner }}
  repository: "pytorch/builder"
- ref: ${{ inputs.ref || github.ref }}
+ ref: ${{ github.ref }}
  job-name: ${{ matrix.build_name }}
  binary-matrix: ${{ toJSON(matrix) }}
  script: |
@@ -124,6 +124,7 @@ jobs:
  export RELEASE_VERSION=${{ inputs.version }}
  export USE_FORCE_REINSTALL=${{ inputs.use-force-reinstall }}
  export USE_EXTRA_INDEX_URL=${{ inputs.use-extra-index-url }}
+ export USE_META_CDN=${{ inputs.use-meta-cdn }}
  export USE_VERSION_SET=${{ inputs.use-version-set }}
 
  if [[ ${{ matrix.package_type }} == "conda" ]]; then

diff --git a/.github/workflows/validate-windows-binaries.yml b/.github/workflows/validate-windows-binaries.yml
@@ -7,11 +7,6 @@ on:
  description: "Channel to use (nightly, test, release, all)"
  required: true
  type: string
- ref:
- description: 'Reference to checkout, defaults to empty'
- default: ""
- required: false
- type: string
  torchonly:
  description: 'Validate torchonly'
  default: false
@@ -42,6 +37,11 @@ on:
  default: false
  required: false
  type: boolean
+ use-meta-cdn:
+ description: 'Use meta cdn for installing pip binaries'
+ default: false
+ required: false
+ type: boolean
  use-extra-index-url:
  description: 'Use extra-index url for pip tests'
  default: false
@@ -58,11 +58,6 @@ on:
  - nightly
  - test
  - all
- ref:
- description: 'Reference to checkout, defaults to empty'
- default: ""
- required: false
- type: string
  torchonly:
  description: 'Validate torchonly'
  default: false
@@ -88,6 +83,11 @@ on:
  default: false
  required: false
  type: boolean
+ use-meta-cdn:
+ description: 'Use meta cdn for installing pip binaries'
+ default: false
+ required: false
+ type: boolean
  use-extra-index-url:
  description: 'Use extra-index url for pip tests'
  default: false
@@ -113,7 +113,7 @@ jobs:
  with:
  runner: ${{ matrix.package_type == 'libtorch' && 'windows.4xlarge' || matrix.validation_runner }}
  repository: "pytorch/builder"
- ref: ${{ inputs.ref || github.ref }}
+ ref: ${{ github.ref }}
  job-name: ${{ matrix.build_name }}
  binary-matrix: ${{ toJSON(matrix) }}
  timeout: 60
@@ -125,6 +125,7 @@ jobs:
  export RELEASE_VERSION=${{ inputs.version }}
  export USE_FORCE_REINSTALL=${{ inputs.use-force-reinstall }}
  export USE_EXTRA_INDEX_URL=${{ inputs.use-extra-index-url }}
+ export USE_META_CDN=${{ inputs.use-meta-cdn }}
  export USE_VERSION_SET=${{ inputs.use-version-set }}
  if [[ ${USE_VERSION_SET} == 'true' ]]; then
  export VISION_RELEASE_VERSION=${{ fromJson(inputs.release-matrix).torchvision }}