From bba4202661de519f5f85a7da3d9c520238b07730 Mon Sep 17 00:00:00 2001 From: peter Date: Mon, 24 Dec 2018 14:29:56 +0800 Subject: [PATCH 01/30] Refactor the build config file --- windows/azure-pipelines-min.yml | 61 +++++ windows/azure-pipelines.yml | 249 ++++-------------- windows/templates/auth_task.yml | 17 ++ windows/templates/build_conda.yml | 9 + windows/templates/build_task.yml | 119 +++++++++ windows/templates/build_wheels.yml | 12 + windows/templates/linux_build_task.yml | 37 +++ windows/templates/publish_test_results.yml | 6 + windows/templates/setup_env_for_msagent.yml | 26 ++ windows/templates/setup_nightly_variables.yml | 7 + windows/templates/upload_to_conda.yml | 10 + windows/templates/vsts_auth.yml | 8 + 12 files changed, 367 insertions(+), 194 deletions(-) create mode 100644 windows/azure-pipelines-min.yml create mode 100644 windows/templates/auth_task.yml create mode 100644 windows/templates/build_conda.yml create mode 100644 windows/templates/build_task.yml create mode 100644 windows/templates/build_wheels.yml create mode 100644 windows/templates/linux_build_task.yml create mode 100644 windows/templates/publish_test_results.yml create mode 100644 windows/templates/setup_env_for_msagent.yml create mode 100644 windows/templates/setup_nightly_variables.yml create mode 100644 windows/templates/upload_to_conda.yml create mode 100644 windows/templates/vsts_auth.yml diff --git a/windows/azure-pipelines-min.yml b/windows/azure-pipelines-min.yml new file mode 100644 index 000000000..ba42b70cb --- /dev/null +++ b/windows/azure-pipelines-min.yml @@ -0,0 +1,61 @@ + +# Turn off auto builds for commits +trigger: none +pr: none + +variables: + # pytorch.build.version: '1.0.0' + # pytorch.build.number: 1 + # pytorch.branch: master + # pytorch.repo: pytorch + build.linux.enabled: false + build.cpu.enabled: true + build.cuda.enabled: true + build.wheels.enabled: true + build.conda.enabled: true + self.hosted.agent.cpu: false + self.hosted.agent.cuda: true + +jobs: +- template: templates/auth_task.yml + +- template: templates/build_task.yml + parameters: + package: 'wheels' + spec: 'cpu' + jobDesc: 'Wheels' + packageDesc: 'CPU' + jobEnabled: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) + msagent: true + +- template: templates/build_task.yml + parameters: + package: 'conda' + spec: 'cpu' + jobDesc: 'Conda' + packageDesc: 'CPU' + jobEnabled: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) + msagent: true + +- template: templates/build_task.yml + parameters: + package: 'wheels' + spec: 'cuda' + jobDesc: 'Wheels' + packageDesc: 'CUDA' + jobEnabled: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) + msagent: false + +- template: templates/build_task.yml + parameters: + package: 'conda' + spec: 'cuda' + jobDesc: 'Conda' + packageDesc: 'CUDA' + jobEnabled: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) + msagent: false + +- template: templates/linux_build_task.yml + parameters: + jobEnabled: and(and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.linux.enabled'], 'true')), eq(variables['build.conda.enabled'], 'true')) + msagent: false diff --git a/windows/azure-pipelines.yml b/windows/azure-pipelines.yml index fe5190c65..e886737e4 100644 --- a/windows/azure-pipelines.yml +++ b/windows/azure-pipelines.yml @@ -4,7 +4,7 @@ trigger: none pr: none variables: - # pytorch.build.version: '1.0' + # pytorch.build.version: '1.0.0' # pytorch.build.number: 1 # pytorch.branch: master # pytorch.repo: pytorch @@ -32,10 +32,9 @@ jobs: - checkout: self clean: true - - script: 'call windows/internal/auth.bat' - displayName: 'Sign in to Azure Pipelines' - env: - VSTS_AUTH: $(vsts_auth) + - template: templates/vsts_auth.yml + parameters: + auth: $(vsts_auth) - job: 'Windows_CPU_Wheels_Build' timeoutInMinutes: 0 @@ -64,48 +63,19 @@ jobs: - checkout: self clean: true - - task: BatchScript@1 - condition: eq(variables['self.hosted.agent.cpu'], 'false') - displayName: 'Install VS 2017' - inputs: - filename: 'windows/internal/vs_install.bat' - - modifyEnvironment: true - - - task: BatchScript@1 - condition: eq(variables['self.hosted.agent.cpu'], 'false') - enabled: false - displayName: 'Environment fix' - inputs: - filename: 'windows/internal/env_fix.bat' - - modifyEnvironment: false - - - task: BatchScript@1 - condition: eq(variables['self.hosted.agent.cpu'], 'false') - displayName: 'Install 7Zip' - inputs: - filename: 'windows/internal/7z_install.bat' - - modifyEnvironment: true - - - task: BatchScript@1 - displayName: 'Setup nightly variables' - inputs: - filename: 'windows/internal/nightly_defaults.bat' + - template: templates/setup_env_for_msagent.yml + parameters: + msagent: $(self.hosted.agent.cpu) - modifyEnvironment: true + - template: templates/setup_nightly_variables.yml - - script: 'call windows/build_pytorch.bat %CUDA_VERSION% %PYTORCH_BUILD_VERSION% %PYTORCH_BUILD_NUMBER%' - displayName: Build - env: - MAX_JOBS: 2 + - template: templates/build_wheels.yml + parameters: + msagent: $(self.hosted.agent.cpu) - - task: PublishTestResults@2 # No test results to publish - inputs: - testResultsFiles: 'windows/pytorch/test/**/*.xml' - testRunTitle: 'Publish test results' - enabled: false + - template: templates/publish_test_results.yml + parameters: + msagent: $(self.hosted.agent.cpu) - template: templates/upload_to_s3.yml parameters: @@ -142,57 +112,24 @@ jobs: - checkout: self clean: true - - task: BatchScript@1 - condition: eq(variables['self.hosted.agent.cpu'], 'false') - displayName: 'Install VS 2017' - inputs: - filename: 'windows/internal/vs_install.bat' - - modifyEnvironment: true - - - task: BatchScript@1 - condition: eq(variables['self.hosted.agent.cpu'], 'false') - enabled: false - displayName: 'Environment fix' - inputs: - filename: 'windows/internal/env_fix.bat' - - modifyEnvironment: false - - - task: BatchScript@1 - condition: eq(variables['self.hosted.agent.cpu'], 'false') - displayName: 'Install 7Zip' - inputs: - filename: 'windows/internal/7z_install.bat' - - modifyEnvironment: true - - - task: BatchScript@1 - displayName: 'Setup nightly variables' - inputs: - filename: 'windows/internal/nightly_defaults.bat' - - modifyEnvironment: true + - template: templates/setup_env_for_msagent.yml + parameters: + msagent: $(self.hosted.agent.cpu) - - script: 'if not exist %PYTORCH_FINAL_PACKAGE_DIR% mkdir %PYTORCH_FINAL_PACKAGE_DIR%' - displayName: 'Create final package directory' + - template: templates/setup_nightly_variables.yml - - bash: './conda/build_pytorch.sh $CUDA_VERSION $PYTORCH_BUILD_VERSION $PYTORCH_BUILD_NUMBER' - displayName: 'Build' - env: - MAX_JOBS: 2 + - template: templates/build_conda.yml + parameters: + msagent: $(self.hosted.agent.cpu) - - task: PublishTestResults@2 - inputs: - testResultsFiles: 'windows/pytorch/test/**/*.xml' - testRunTitle: 'Publish test results' - enabled: false + - template: templates/publish_test_results.yml + parameters: + msagent: $(self.hosted.agent.cpu) - - script: 'call windows/internal/upload.bat' - displayName: 'Upload packages to Anaconda Cloud' - env: - PYTORCH_ANACONDA_USERNAME: $(peterjc_conda_username) - PYTORCH_ANACONDA_PASSWORD: $(peterjc_conda_password) + - template: templates/upload_to_conda.yml + parameters: + user: $(peterjc_conda_username) + pass: $(peterjc_conda_password) - job: 'Linux_CPU_Conda_Build' timeoutInMinutes: 0 @@ -224,13 +161,9 @@ jobs: inputs: packageSpecs: 'conda-build' - - script: 'mkdir %PYTORCH_FINAL_PACKAGE_DIR%' - displayName: 'Create final package directory' - - - bash: './conda/build_pytorch.sh $CUDA_VERSION $PYTORCH_BUILD_VERSION $PYTORCH_BUILD_NUMBER' - displayName: 'Build' - env: - MAX_JOBS: 2 + - template: templates/build_conda.yml + parameters: + msagent: $(self.hosted.agent.cpu) - job: 'Windows_CUDA_Wheels_Build' timeoutInMinutes: 0 @@ -288,53 +221,19 @@ jobs: - checkout: self clean: true - - task: BatchScript@1 - condition: eq(variables['self.hosted.agent.cuda'], 'false') - displayName: 'Install VS 2017' - inputs: - filename: 'windows/internal/vs_install.bat' - - modifyEnvironment: true - - - task: BatchScript@1 - condition: eq(variables['self.hosted.agent.cuda'], 'false') - displayName: 'Install 7Zip' - inputs: - filename: 'windows/internal/7z_install.bat' - - modifyEnvironment: true - - - task: BatchScript@1 - condition: eq(variables['self.hosted.agent.cuda'], 'false') - displayName: 'Install CUDA' - inputs: - filename: 'windows/internal/cuda_install.bat' - - modifyEnvironment: true - - - task: BatchScript@1 - condition: eq(variables['self.hosted.agent.cuda'], 'false') - displayName: 'Environment fix' - inputs: - filename: 'windows/internal/env_fix.bat' - - modifyEnvironment: false - - - task: BatchScript@1 - displayName: 'Setup nightly variables' - inputs: - filename: 'windows/internal/nightly_defaults.bat' + - template: templates/setup_env_for_msagent.yml + parameters: + msagent: $(self.hosted.agent.cpu) - modifyEnvironment: true + - template: templates/setup_nightly_variables.yml - - script: 'call windows/build_pytorch.bat %CUDA_VERSION% %PYTORCH_BUILD_VERSION% %PYTORCH_BUILD_NUMBER%' - displayName: Build + - template: templates/build_wheels.yml + parameters: + msagent: $(self.hosted.agent.cpu) - - task: PublishTestResults@2 # No test results to publish - inputs: - testResultsFiles: 'windows/pytorch/test/**/*.xml' - testRunTitle: 'Publish test results' - enabled: false + - template: templates/publish_test_results.yml + parameters: + msagent: $(self.hosted.agent.cpu) - template: templates/upload_to_s3.yml parameters: @@ -390,59 +289,21 @@ jobs: - checkout: self clean: true - - task: BatchScript@1 - condition: eq(variables['self.hosted.agent.cuda'], 'false') - displayName: 'Install VS 2017' - inputs: - filename: 'windows/internal/vs_install.bat' - - modifyEnvironment: true - - - task: BatchScript@1 - condition: eq(variables['self.hosted.agent.cuda'], 'false') - displayName: 'Install 7Zip' - inputs: - filename: 'windows/internal/7z_install.bat' - - modifyEnvironment: true - - - task: BatchScript@1 - condition: eq(variables['self.hosted.agent.cuda'], 'false') - displayName: 'Install CUDA' - inputs: - filename: 'windows/internal/cuda_install.bat' - - modifyEnvironment: true - - - task: BatchScript@1 - condition: eq(variables['self.hosted.agent.cuda'], 'false') - displayName: 'Environment fix' - inputs: - filename: 'windows/internal/env_fix.bat' - - modifyEnvironment: false - - - task: BatchScript@1 - displayName: 'Setup nightly variables' - inputs: - filename: 'windows/internal/nightly_defaults.bat' + - template: templates/setup_env_for_msagent.yml + parameters: + msagent: $(self.hosted.agent.cpu) - modifyEnvironment: true + - template: templates/setup_nightly_variables.yml - - script: 'if not exist %PYTORCH_FINAL_PACKAGE_DIR% mkdir %PYTORCH_FINAL_PACKAGE_DIR%' - displayName: 'Create final package directory' + - template: templates/build_conda.yml + parameters: + msagent: $(self.hosted.agent.cpu) - - bash: './conda/build_pytorch.sh $CUDA_VERSION $PYTORCH_BUILD_VERSION $PYTORCH_BUILD_NUMBER' - displayName: 'Build' + - template: templates/publish_test_results.yml + parameters: + msagent: $(self.hosted.agent.cpu) - - task: PublishTestResults@2 - inputs: - testResultsFiles: 'windows/pytorch/test/**/*.xml' - testRunTitle: 'Publish test results' - enabled: false - - - script: 'call windows/internal/upload.bat' - displayName: 'Upload packages to Anaconda Cloud' - env: - PYTORCH_ANACONDA_USERNAME: $(peterjc_conda_username) - PYTORCH_ANACONDA_PASSWORD: $(peterjc_conda_password) + - template: templates/upload_to_conda.yml + parameters: + user: $(peterjc_conda_username) + pass: $(peterjc_conda_password) diff --git a/windows/templates/auth_task.yml b/windows/templates/auth_task.yml new file mode 100644 index 000000000..ece66412f --- /dev/null +++ b/windows/templates/auth_task.yml @@ -0,0 +1,17 @@ +jobs: +- job: 'VSTS_Auth_Task' + timeoutInMinutes: 5 + cancelTimeoutInMinutes: 5 + variables: + - group: 'peterjc-vsts-token' + + pool: + vmImage: 'win1803' + + steps: + - checkout: self + clean: true + + - template: vsts_auth.yml + parameters: + auth: $(vsts_auth) diff --git a/windows/templates/build_conda.yml b/windows/templates/build_conda.yml new file mode 100644 index 000000000..e59ffee09 --- /dev/null +++ b/windows/templates/build_conda.yml @@ -0,0 +1,9 @@ +parameters: + msagent: false + +steps: + - script: 'call windows/build_pytorch.bat %CUDA_VERSION% %PYTORCH_BUILD_VERSION% %PYTORCH_BUILD_NUMBER%' + displayName: Build + env: + ${{ if eq(parameters.msagent, 'true') }}: + MAX_JOBS: 2 diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml new file mode 100644 index 000000000..fa61310fd --- /dev/null +++ b/windows/templates/build_task.yml @@ -0,0 +1,119 @@ +parameters: + package: '' + spec: '' + jobDesc: '' + packageDesc: '' + jobEnabled: '' + msagent: '' + +- job: 'Windows_${{ parameters.jobDesc }}_${{ parameters.packageDesc }}_Build' + timeoutInMinutes: 0 + cancelTimeoutInMinutes: 5 + condition: ${{ and(parameters.jobEnabled, 'true') }} + variables: + - name: CUDA_VERSION + value: ${{ parameters.cudaVer }} + - name: USE_SCCACHE + value: ${{ eq(parameters.msagent, 'false') - 0 }} + - ${{ if eq(parameters.package, 'conda') }}: + - group: peterjc_anaconda_token + - name: PYTORCH_FINAL_PACKAGE_DIR + value: '$(Build.Repository.LocalPath)\windows\output' + + strategy: + maxParallel: 10 + matrix: + PY3.5: + DESIRED_PYTHON: 3.5 + PY3.6: + DESIRED_PYTHON: 3.6 + PY3.7: + DESIRED_PYTHON: 3.7 + ${{ if eq(parameters.package, 'wheels') }}: + LIBTORCH: + DESIRED_PYTHON: 3 + BUILD_PYTHONLESS: 1 + ${{ if neq(parameters.spec, 'cpu') }}: + PY3.5_90: + DESIRED_PYTHON: 3.5 + CUDA_VERSION: 90 + PY3.6_90: + DESIRED_PYTHON: 3.6 + CUDA_VERSION: 90 + PY3.7_90: + DESIRED_PYTHON: 3.7 + CUDA_VERSION: 90 + PY3.5_80: + DESIRED_PYTHON: 3.5 + CUDA_VERSION: 80 + PY3.6_80: + DESIRED_PYTHON: 3.6 + CUDA_VERSION: 80 + PY3.7_80: + DESIRED_PYTHON: 3.7 + CUDA_VERSION: 80 + PY3.5_100: + DESIRED_PYTHON: 3.5 + CUDA_VERSION: 100 + PY3.6_100: + DESIRED_PYTHON: 3.6 + CUDA_VERSION: 100 + PY3.7_100: + DESIRED_PYTHON: 3.7 + CUDA_VERSION: 100 + ${{ if and(neq(parameters.spec, 'cpu'), eq(parameters.package, 'wheels')) }}: + LIBTORCH_80: + DESIRED_PYTHON: 3 + BUILD_PYTHONLESS: 1 + CUDA_VERSION: 80 + LIBTORCH_90: + DESIRED_PYTHON: 3 + BUILD_PYTHONLESS: 1 + CUDA_VERSION: 90 + LIBTORCH_100: + DESIRED_PYTHON: 3 + BUILD_PYTHONLESS: 1 + CUDA_VERSION: 100 + + pool: + ${{ if eq(parameters.msagent, 'true') }}: + vmImage: 'win1803' + ${{ if eq(parameters.msagent, 'false') }}: + name: 'pytorch' + + steps: + - checkout: self + clean: true + + - template: setup_env_for_msagent.yml + parameters: + msagent: ${{ parameters.msagent }} + + - template: setup_nightly_variables.yml + +- ${{ if eq(parameters.package, 'wheels') }}: + - template: build_wheels.yml + parameters: + msagent: ${{ parameters.msagent }} + +- ${{ if eq(parameters.package, 'conda') }}: + - template: build_conda.yml + parameters: + msagent: ${{ parameters.msagent }} + +- ${{ if always() }}: + - template: publish_test_results.yml + parameters: + msagent: ${{ parameters.msagent }} + +- ${{ if eq(parameters.package, 'wheels') }}: + - template: upload_to_s3.yml + parameters: + cuVer: 'cu$(CUDA_VERSION)' + cudaVer: 'cuda$(CUDA_VERSION)' + +- ${{ if eq(parameters.package, 'conda') }}: + - template: upload_to_conda.yml + parameters: + user: $(peterjc_conda_username) + pass: $(peterjc_conda_password) diff --git a/windows/templates/build_wheels.yml b/windows/templates/build_wheels.yml new file mode 100644 index 000000000..4aa7e8403 --- /dev/null +++ b/windows/templates/build_wheels.yml @@ -0,0 +1,12 @@ +parameters: + msagent: false + +steps: + - script: 'if not exist %PYTORCH_FINAL_PACKAGE_DIR% mkdir %PYTORCH_FINAL_PACKAGE_DIR%' + displayName: 'Create final package directory' + + - script: 'call windows/build_pytorch.bat %CUDA_VERSION% %PYTORCH_BUILD_VERSION% %PYTORCH_BUILD_NUMBER%' + displayName: Build + env: + ${{ if eq(parameters.msagent, 'true') }}: + MAX_JOBS: 2 diff --git a/windows/templates/linux_build_task.yml b/windows/templates/linux_build_task.yml new file mode 100644 index 000000000..97d038f46 --- /dev/null +++ b/windows/templates/linux_build_task.yml @@ -0,0 +1,37 @@ +parameters: + jobEnabled: '' + msagent: '' + +- job: 'Linux_CPU_Conda_Build' + timeoutInMinutes: 0 + cancelTimeoutInMinutes: 5 + condition: ${{ and(parameters.jobEnabled, 'true') }} + variables: + CUDA_VERSION: cpu + TORCH_CONDA_BUILD_FOLDER: pytorch-nightly + PYTORCH_FINAL_PACKAGE_DIR: '$(Build.Repository.LocalPath)/output' + + strategy: + maxParallel: 10 + matrix: + PY3.5: + DESIRED_PYTHON: 3.5 + + pool: + vmImage: 'ubuntu-16.04' + + steps: + - checkout: self + clean: true + + - script: 'sudo apt-get install p7zip-full' + displayName: 'Install 7Zip' + + - task: CondaEnvironment@1 + displayName: 'Install conda-build' + inputs: + packageSpecs: 'conda-build' + + - template: templates/build_conda.yml + parameters: + msagent: ${{ parameters.msagent }} diff --git a/windows/templates/publish_test_results.yml b/windows/templates/publish_test_results.yml new file mode 100644 index 000000000..f3e89cf95 --- /dev/null +++ b/windows/templates/publish_test_results.yml @@ -0,0 +1,6 @@ + steps: + - task: PublishTestResults@2 # No test results to publish + inputs: + testResultsFiles: 'windows/pytorch/test/**/*.xml' + testRunTitle: 'Publish test results' + enabled: false \ No newline at end of file diff --git a/windows/templates/setup_env_for_msagent.yml b/windows/templates/setup_env_for_msagent.yml new file mode 100644 index 000000000..a8d35692b --- /dev/null +++ b/windows/templates/setup_env_for_msagent.yml @@ -0,0 +1,26 @@ +parameters: + msagent: false + +steps: +- ${{ if eq(parameters.msagent, 'true') }}: + - task: BatchScript@1 + displayName: 'Install VS 2017' + inputs: + filename: 'windows/internal/vs_install.bat' + + modifyEnvironment: true + + - task: BatchScript@1 + enabled: false + displayName: 'Environment fix' + inputs: + filename: 'windows/internal/env_fix.bat' + + modifyEnvironment: false + + - task: BatchScript@1 + displayName: 'Install 7Zip' + inputs: + filename: 'windows/internal/7z_install.bat' + + modifyEnvironment: true diff --git a/windows/templates/setup_nightly_variables.yml b/windows/templates/setup_nightly_variables.yml new file mode 100644 index 000000000..24f8cd2fc --- /dev/null +++ b/windows/templates/setup_nightly_variables.yml @@ -0,0 +1,7 @@ +steps: + - task: BatchScript@1 + displayName: 'Setup nightly variables' + inputs: + filename: 'windows/internal/nightly_defaults.bat' + + modifyEnvironment: true diff --git a/windows/templates/upload_to_conda.yml b/windows/templates/upload_to_conda.yml new file mode 100644 index 000000000..3ea43a556 --- /dev/null +++ b/windows/templates/upload_to_conda.yml @@ -0,0 +1,10 @@ +parameters: + user: '' + pass: '' + +steps: + - script: 'call windows/internal/upload.bat' + displayName: 'Upload packages to Anaconda Cloud' + env: + PYTORCH_ANACONDA_USERNAME: ${{ parameters.user }} + PYTORCH_ANACONDA_PASSWORD: ${{ parameters.pass }} \ No newline at end of file diff --git a/windows/templates/vsts_auth.yml b/windows/templates/vsts_auth.yml new file mode 100644 index 000000000..e83a444f3 --- /dev/null +++ b/windows/templates/vsts_auth.yml @@ -0,0 +1,8 @@ +parameters: + auth: '' + +steps: + - script: 'call windows/internal/auth.bat' + displayName: 'Sign in to Azure Pipelines' + env: + VSTS_AUTH: ${{ parameters.auth }} From d4530688915cc0e6d06b069fa9b86ccbeafc2357 Mon Sep 17 00:00:00 2001 From: peter Date: Mon, 24 Dec 2018 14:37:32 +0800 Subject: [PATCH 02/30] Minor fixes --- windows/azure-pipelines-min.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/windows/azure-pipelines-min.yml b/windows/azure-pipelines-min.yml index ba42b70cb..e6e795e5e 100644 --- a/windows/azure-pipelines-min.yml +++ b/windows/azure-pipelines-min.yml @@ -13,8 +13,8 @@ variables: build.cuda.enabled: true build.wheels.enabled: true build.conda.enabled: true - self.hosted.agent.cpu: false - self.hosted.agent.cuda: true + ms.hosted.agent.cpu: true + ms.hosted.agent.cuda: false jobs: - template: templates/auth_task.yml @@ -26,7 +26,7 @@ jobs: jobDesc: 'Wheels' packageDesc: 'CPU' jobEnabled: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) - msagent: true + msagent: $(ms.hosted.agent.cpu) - template: templates/build_task.yml parameters: @@ -35,7 +35,7 @@ jobs: jobDesc: 'Conda' packageDesc: 'CPU' jobEnabled: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) - msagent: true + msagent: $(ms.hosted.agent.cpu) - template: templates/build_task.yml parameters: @@ -44,7 +44,7 @@ jobs: jobDesc: 'Wheels' packageDesc: 'CUDA' jobEnabled: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) - msagent: false + msagent: $(ms.hosted.agent.cuda) - template: templates/build_task.yml parameters: @@ -53,9 +53,9 @@ jobs: jobDesc: 'Conda' packageDesc: 'CUDA' jobEnabled: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) - msagent: false + msagent: $(ms.hosted.agent.cuda) - template: templates/linux_build_task.yml parameters: jobEnabled: and(and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.linux.enabled'], 'true')), eq(variables['build.conda.enabled'], 'true')) - msagent: false + msagent: $(ms.hosted.agent.cpu) From fa773ed8a336d930502f3600bf72c82e161294f7 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 14:49:54 +0800 Subject: [PATCH 03/30] Minor fixes --- windows/azure-pipelines-legacy.yml | 309 ++++++++++++++++++++++++++++ windows/azure-pipelines-min.yml | 61 ------ windows/azure-pipelines.yml | 316 ++++------------------------- windows/templates/build_task.yml | 79 ++++---- 4 files changed, 383 insertions(+), 382 deletions(-) create mode 100644 windows/azure-pipelines-legacy.yml delete mode 100644 windows/azure-pipelines-min.yml diff --git a/windows/azure-pipelines-legacy.yml b/windows/azure-pipelines-legacy.yml new file mode 100644 index 000000000..e886737e4 --- /dev/null +++ b/windows/azure-pipelines-legacy.yml @@ -0,0 +1,309 @@ + +# Turn off auto builds for commits +trigger: none +pr: none + +variables: + # pytorch.build.version: '1.0.0' + # pytorch.build.number: 1 + # pytorch.branch: master + # pytorch.repo: pytorch + build.linux.enabled: false + build.cpu.enabled: true + build.cuda.enabled: true + build.wheels.enabled: true + build.conda.enabled: true + use.sccache.cpu: 0 + use.sccache.cuda: 1 + self.hosted.agent.cpu: false + self.hosted.agent.cuda: true + +jobs: +- job: 'VSTS_Auth_Task' + timeoutInMinutes: 5 + cancelTimeoutInMinutes: 5 + variables: + - group: 'peterjc-vsts-token' + + pool: + vmImage: 'win1803' + + steps: + - checkout: self + clean: true + + - template: templates/vsts_auth.yml + parameters: + auth: $(vsts_auth) + +- job: 'Windows_CPU_Wheels_Build' + timeoutInMinutes: 0 + cancelTimeoutInMinutes: 5 + condition: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) + variables: + CUDA_VERSION: cpu + USE_SCCACHE: $(use.sccache.cpu) + strategy: + maxParallel: 10 + matrix: + PY3.5: + DESIRED_PYTHON: 3.5 + PY3.6: + DESIRED_PYTHON: 3.6 + PY3.7: + DESIRED_PYTHON: 3.7 + LIBTORCH: + DESIRED_PYTHON: 3 + BUILD_PYTHONLESS: 1 + + pool: + vmImage: 'win1803' + + steps: + - checkout: self + clean: true + + - template: templates/setup_env_for_msagent.yml + parameters: + msagent: $(self.hosted.agent.cpu) + + - template: templates/setup_nightly_variables.yml + + - template: templates/build_wheels.yml + parameters: + msagent: $(self.hosted.agent.cpu) + + - template: templates/publish_test_results.yml + parameters: + msagent: $(self.hosted.agent.cpu) + + - template: templates/upload_to_s3.yml + parameters: + cuVer: 'cpu' + cudaVer: 'cpu' + +- job: 'Windows_CPU_Conda_Build' + timeoutInMinutes: 0 + cancelTimeoutInMinutes: 5 + condition: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) + variables: + - group: peterjc_anaconda_token + - name: CUDA_VERSION + value: cpu + - name: USE_SCCACHE + value: $(use.sccache.cpu) + - name: PYTORCH_FINAL_PACKAGE_DIR + value: '$(Build.Repository.LocalPath)\windows\output' + + strategy: + maxParallel: 10 + matrix: + PY3.5: + DESIRED_PYTHON: 3.5 + PY3.6: + DESIRED_PYTHON: 3.6 + PY3.7: + DESIRED_PYTHON: 3.7 + + pool: + vmImage: 'win1803' + + steps: + - checkout: self + clean: true + + - template: templates/setup_env_for_msagent.yml + parameters: + msagent: $(self.hosted.agent.cpu) + + - template: templates/setup_nightly_variables.yml + + - template: templates/build_conda.yml + parameters: + msagent: $(self.hosted.agent.cpu) + + - template: templates/publish_test_results.yml + parameters: + msagent: $(self.hosted.agent.cpu) + + - template: templates/upload_to_conda.yml + parameters: + user: $(peterjc_conda_username) + pass: $(peterjc_conda_password) + +- job: 'Linux_CPU_Conda_Build' + timeoutInMinutes: 0 + cancelTimeoutInMinutes: 5 + condition: and(and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.linux.enabled'], 'true')), eq(variables['build.conda.enabled'], 'true')) + variables: + CUDA_VERSION: cpu + TORCH_CONDA_BUILD_FOLDER: pytorch-nightly + PYTORCH_FINAL_PACKAGE_DIR: '$(Build.Repository.LocalPath)/output' + + strategy: + maxParallel: 10 + matrix: + PY3.5: + DESIRED_PYTHON: 3.5 + + pool: + vmImage: 'ubuntu-16.04' + + steps: + - checkout: self + clean: true + + - script: 'sudo apt-get install p7zip-full' + displayName: 'Install 7Zip' + + - task: CondaEnvironment@1 + displayName: 'Install conda-build' + inputs: + packageSpecs: 'conda-build' + + - template: templates/build_conda.yml + parameters: + msagent: $(self.hosted.agent.cpu) + +- job: 'Windows_CUDA_Wheels_Build' + timeoutInMinutes: 0 + cancelTimeoutInMinutes: 5 + condition: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) + variables: + USE_SCCACHE: $(use.sccache.cuda) + strategy: + maxParallel: 10 + matrix: + PY3.5_90: + DESIRED_PYTHON: 3.5 + CUDA_VERSION: 90 + PY3.6_90: + DESIRED_PYTHON: 3.6 + CUDA_VERSION: 90 + PY3.7_90: + DESIRED_PYTHON: 3.7 + CUDA_VERSION: 90 + LIBTORCH_90: + DESIRED_PYTHON: 3 + BUILD_PYTHONLESS: 1 + CUDA_VERSION: 90 + PY3.5_80: + DESIRED_PYTHON: 3.5 + CUDA_VERSION: 80 + PY3.6_80: + DESIRED_PYTHON: 3.6 + CUDA_VERSION: 80 + PY3.7_80: + DESIRED_PYTHON: 3.7 + CUDA_VERSION: 80 + LIBTORCH_80: + DESIRED_PYTHON: 3 + BUILD_PYTHONLESS: 1 + CUDA_VERSION: 80 + PY3.5_100: + DESIRED_PYTHON: 3.5 + CUDA_VERSION: 100 + PY3.6_100: + DESIRED_PYTHON: 3.6 + CUDA_VERSION: 100 + PY3.7_100: + DESIRED_PYTHON: 3.7 + CUDA_VERSION: 100 + LIBTORCH_100: + DESIRED_PYTHON: 3 + BUILD_PYTHONLESS: 1 + CUDA_VERSION: 100 + + pool: + name: 'pytorch' + + steps: + - checkout: self + clean: true + + - template: templates/setup_env_for_msagent.yml + parameters: + msagent: $(self.hosted.agent.cpu) + + - template: templates/setup_nightly_variables.yml + + - template: templates/build_wheels.yml + parameters: + msagent: $(self.hosted.agent.cpu) + + - template: templates/publish_test_results.yml + parameters: + msagent: $(self.hosted.agent.cpu) + + - template: templates/upload_to_s3.yml + parameters: + cuVer: 'cu$(CUDA_VERSION)' + cudaVer: 'cuda$(CUDA_VERSION)' + +- job: 'Windows_CUDA_Conda_Build' + timeoutInMinutes: 0 + condition: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) + cancelTimeoutInMinutes: 5 + variables: + - group: peterjc_anaconda_token + - name: USE_SCCACHE + value: $(use.sccache.cuda) + - name: PYTORCH_FINAL_PACKAGE_DIR + value: '$(Build.Repository.LocalPath)\windows\output' + + strategy: + maxParallel: 10 + matrix: + PY3.5_90: + DESIRED_PYTHON: 3.5 + CUDA_VERSION: 90 + PY3.6_90: + DESIRED_PYTHON: 3.6 + CUDA_VERSION: 90 + PY3.7_90: + DESIRED_PYTHON: 3.7 + CUDA_VERSION: 90 + PY3.5_80: + DESIRED_PYTHON: 3.5 + CUDA_VERSION: 80 + PY3.6_80: + DESIRED_PYTHON: 3.6 + CUDA_VERSION: 80 + PY3.7_80: + DESIRED_PYTHON: 3.7 + CUDA_VERSION: 80 + PY3.5_100: + DESIRED_PYTHON: 3.5 + CUDA_VERSION: 100 + PY3.6_100: + DESIRED_PYTHON: 3.6 + CUDA_VERSION: 100 + PY3.7_100: + DESIRED_PYTHON: 3.7 + CUDA_VERSION: 100 + + pool: + name: 'pytorch' + + steps: + - checkout: self + clean: true + + - template: templates/setup_env_for_msagent.yml + parameters: + msagent: $(self.hosted.agent.cpu) + + - template: templates/setup_nightly_variables.yml + + - template: templates/build_conda.yml + parameters: + msagent: $(self.hosted.agent.cpu) + + - template: templates/publish_test_results.yml + parameters: + msagent: $(self.hosted.agent.cpu) + + - template: templates/upload_to_conda.yml + parameters: + user: $(peterjc_conda_username) + pass: $(peterjc_conda_password) diff --git a/windows/azure-pipelines-min.yml b/windows/azure-pipelines-min.yml deleted file mode 100644 index e6e795e5e..000000000 --- a/windows/azure-pipelines-min.yml +++ /dev/null @@ -1,61 +0,0 @@ - -# Turn off auto builds for commits -trigger: none -pr: none - -variables: - # pytorch.build.version: '1.0.0' - # pytorch.build.number: 1 - # pytorch.branch: master - # pytorch.repo: pytorch - build.linux.enabled: false - build.cpu.enabled: true - build.cuda.enabled: true - build.wheels.enabled: true - build.conda.enabled: true - ms.hosted.agent.cpu: true - ms.hosted.agent.cuda: false - -jobs: -- template: templates/auth_task.yml - -- template: templates/build_task.yml - parameters: - package: 'wheels' - spec: 'cpu' - jobDesc: 'Wheels' - packageDesc: 'CPU' - jobEnabled: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) - msagent: $(ms.hosted.agent.cpu) - -- template: templates/build_task.yml - parameters: - package: 'conda' - spec: 'cpu' - jobDesc: 'Conda' - packageDesc: 'CPU' - jobEnabled: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) - msagent: $(ms.hosted.agent.cpu) - -- template: templates/build_task.yml - parameters: - package: 'wheels' - spec: 'cuda' - jobDesc: 'Wheels' - packageDesc: 'CUDA' - jobEnabled: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) - msagent: $(ms.hosted.agent.cuda) - -- template: templates/build_task.yml - parameters: - package: 'conda' - spec: 'cuda' - jobDesc: 'Conda' - packageDesc: 'CUDA' - jobEnabled: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) - msagent: $(ms.hosted.agent.cuda) - -- template: templates/linux_build_task.yml - parameters: - jobEnabled: and(and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.linux.enabled'], 'true')), eq(variables['build.conda.enabled'], 'true')) - msagent: $(ms.hosted.agent.cpu) diff --git a/windows/azure-pipelines.yml b/windows/azure-pipelines.yml index e886737e4..e6e795e5e 100644 --- a/windows/azure-pipelines.yml +++ b/windows/azure-pipelines.yml @@ -13,297 +13,49 @@ variables: build.cuda.enabled: true build.wheels.enabled: true build.conda.enabled: true - use.sccache.cpu: 0 - use.sccache.cuda: 1 - self.hosted.agent.cpu: false - self.hosted.agent.cuda: true + ms.hosted.agent.cpu: true + ms.hosted.agent.cuda: false jobs: -- job: 'VSTS_Auth_Task' - timeoutInMinutes: 5 - cancelTimeoutInMinutes: 5 - variables: - - group: 'peterjc-vsts-token' +- template: templates/auth_task.yml - pool: - vmImage: 'win1803' - - steps: - - checkout: self - clean: true - - - template: templates/vsts_auth.yml - parameters: - auth: $(vsts_auth) - -- job: 'Windows_CPU_Wheels_Build' - timeoutInMinutes: 0 - cancelTimeoutInMinutes: 5 - condition: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) - variables: - CUDA_VERSION: cpu - USE_SCCACHE: $(use.sccache.cpu) - strategy: - maxParallel: 10 - matrix: - PY3.5: - DESIRED_PYTHON: 3.5 - PY3.6: - DESIRED_PYTHON: 3.6 - PY3.7: - DESIRED_PYTHON: 3.7 - LIBTORCH: - DESIRED_PYTHON: 3 - BUILD_PYTHONLESS: 1 - - pool: - vmImage: 'win1803' - - steps: - - checkout: self - clean: true - - - template: templates/setup_env_for_msagent.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/setup_nightly_variables.yml - - - template: templates/build_wheels.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/publish_test_results.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/upload_to_s3.yml - parameters: - cuVer: 'cpu' - cudaVer: 'cpu' - -- job: 'Windows_CPU_Conda_Build' - timeoutInMinutes: 0 - cancelTimeoutInMinutes: 5 - condition: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) - variables: - - group: peterjc_anaconda_token - - name: CUDA_VERSION - value: cpu - - name: USE_SCCACHE - value: $(use.sccache.cpu) - - name: PYTORCH_FINAL_PACKAGE_DIR - value: '$(Build.Repository.LocalPath)\windows\output' - - strategy: - maxParallel: 10 - matrix: - PY3.5: - DESIRED_PYTHON: 3.5 - PY3.6: - DESIRED_PYTHON: 3.6 - PY3.7: - DESIRED_PYTHON: 3.7 - - pool: - vmImage: 'win1803' - - steps: - - checkout: self - clean: true - - - template: templates/setup_env_for_msagent.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/setup_nightly_variables.yml - - - template: templates/build_conda.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/publish_test_results.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/upload_to_conda.yml - parameters: - user: $(peterjc_conda_username) - pass: $(peterjc_conda_password) - -- job: 'Linux_CPU_Conda_Build' - timeoutInMinutes: 0 - cancelTimeoutInMinutes: 5 - condition: and(and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.linux.enabled'], 'true')), eq(variables['build.conda.enabled'], 'true')) - variables: - CUDA_VERSION: cpu - TORCH_CONDA_BUILD_FOLDER: pytorch-nightly - PYTORCH_FINAL_PACKAGE_DIR: '$(Build.Repository.LocalPath)/output' - - strategy: - maxParallel: 10 - matrix: - PY3.5: - DESIRED_PYTHON: 3.5 - - pool: - vmImage: 'ubuntu-16.04' - - steps: - - checkout: self - clean: true - - - script: 'sudo apt-get install p7zip-full' - displayName: 'Install 7Zip' - - - task: CondaEnvironment@1 - displayName: 'Install conda-build' - inputs: - packageSpecs: 'conda-build' - - - template: templates/build_conda.yml - parameters: - msagent: $(self.hosted.agent.cpu) - -- job: 'Windows_CUDA_Wheels_Build' - timeoutInMinutes: 0 - cancelTimeoutInMinutes: 5 - condition: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) - variables: - USE_SCCACHE: $(use.sccache.cuda) - strategy: - maxParallel: 10 - matrix: - PY3.5_90: - DESIRED_PYTHON: 3.5 - CUDA_VERSION: 90 - PY3.6_90: - DESIRED_PYTHON: 3.6 - CUDA_VERSION: 90 - PY3.7_90: - DESIRED_PYTHON: 3.7 - CUDA_VERSION: 90 - LIBTORCH_90: - DESIRED_PYTHON: 3 - BUILD_PYTHONLESS: 1 - CUDA_VERSION: 90 - PY3.5_80: - DESIRED_PYTHON: 3.5 - CUDA_VERSION: 80 - PY3.6_80: - DESIRED_PYTHON: 3.6 - CUDA_VERSION: 80 - PY3.7_80: - DESIRED_PYTHON: 3.7 - CUDA_VERSION: 80 - LIBTORCH_80: - DESIRED_PYTHON: 3 - BUILD_PYTHONLESS: 1 - CUDA_VERSION: 80 - PY3.5_100: - DESIRED_PYTHON: 3.5 - CUDA_VERSION: 100 - PY3.6_100: - DESIRED_PYTHON: 3.6 - CUDA_VERSION: 100 - PY3.7_100: - DESIRED_PYTHON: 3.7 - CUDA_VERSION: 100 - LIBTORCH_100: - DESIRED_PYTHON: 3 - BUILD_PYTHONLESS: 1 - CUDA_VERSION: 100 - - pool: - name: 'pytorch' - - steps: - - checkout: self - clean: true - - - template: templates/setup_env_for_msagent.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/setup_nightly_variables.yml - - - template: templates/build_wheels.yml +- template: templates/build_task.yml parameters: - msagent: $(self.hosted.agent.cpu) + package: 'wheels' + spec: 'cpu' + jobDesc: 'Wheels' + packageDesc: 'CPU' + jobEnabled: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) + msagent: $(ms.hosted.agent.cpu) - - template: templates/publish_test_results.yml +- template: templates/build_task.yml parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/upload_to_s3.yml - parameters: - cuVer: 'cu$(CUDA_VERSION)' - cudaVer: 'cuda$(CUDA_VERSION)' - -- job: 'Windows_CUDA_Conda_Build' - timeoutInMinutes: 0 - condition: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) - cancelTimeoutInMinutes: 5 - variables: - - group: peterjc_anaconda_token - - name: USE_SCCACHE - value: $(use.sccache.cuda) - - name: PYTORCH_FINAL_PACKAGE_DIR - value: '$(Build.Repository.LocalPath)\windows\output' - - strategy: - maxParallel: 10 - matrix: - PY3.5_90: - DESIRED_PYTHON: 3.5 - CUDA_VERSION: 90 - PY3.6_90: - DESIRED_PYTHON: 3.6 - CUDA_VERSION: 90 - PY3.7_90: - DESIRED_PYTHON: 3.7 - CUDA_VERSION: 90 - PY3.5_80: - DESIRED_PYTHON: 3.5 - CUDA_VERSION: 80 - PY3.6_80: - DESIRED_PYTHON: 3.6 - CUDA_VERSION: 80 - PY3.7_80: - DESIRED_PYTHON: 3.7 - CUDA_VERSION: 80 - PY3.5_100: - DESIRED_PYTHON: 3.5 - CUDA_VERSION: 100 - PY3.6_100: - DESIRED_PYTHON: 3.6 - CUDA_VERSION: 100 - PY3.7_100: - DESIRED_PYTHON: 3.7 - CUDA_VERSION: 100 - - pool: - name: 'pytorch' - - steps: - - checkout: self - clean: true - - - template: templates/setup_env_for_msagent.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/setup_nightly_variables.yml + package: 'conda' + spec: 'cpu' + jobDesc: 'Conda' + packageDesc: 'CPU' + jobEnabled: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) + msagent: $(ms.hosted.agent.cpu) - - template: templates/build_conda.yml +- template: templates/build_task.yml parameters: - msagent: $(self.hosted.agent.cpu) + package: 'wheels' + spec: 'cuda' + jobDesc: 'Wheels' + packageDesc: 'CUDA' + jobEnabled: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) + msagent: $(ms.hosted.agent.cuda) - - template: templates/publish_test_results.yml +- template: templates/build_task.yml parameters: - msagent: $(self.hosted.agent.cpu) + package: 'conda' + spec: 'cuda' + jobDesc: 'Conda' + packageDesc: 'CUDA' + jobEnabled: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) + msagent: $(ms.hosted.agent.cuda) - - template: templates/upload_to_conda.yml +- template: templates/linux_build_task.yml parameters: - user: $(peterjc_conda_username) - pass: $(peterjc_conda_password) + jobEnabled: and(and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.linux.enabled'], 'true')), eq(variables['build.conda.enabled'], 'true')) + msagent: $(ms.hosted.agent.cpu) diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml index fa61310fd..e112fa7cf 100644 --- a/windows/templates/build_task.yml +++ b/windows/templates/build_task.yml @@ -23,13 +23,14 @@ parameters: strategy: maxParallel: 10 matrix: - PY3.5: - DESIRED_PYTHON: 3.5 - PY3.6: - DESIRED_PYTHON: 3.6 - PY3.7: - DESIRED_PYTHON: 3.7 - ${{ if eq(parameters.package, 'wheels') }}: + ${{ if eq(parameters.spec, 'cpu') }}: + PY3.5: + DESIRED_PYTHON: 3.5 + # PY3.6: + # DESIRED_PYTHON: 3.6 + # PY3.7: + # DESIRED_PYTHON: 3.7 + ${{ if and(eq(parameters.package, 'wheels'), eq(parameters.spec, 'cpu')) }}: LIBTORCH: DESIRED_PYTHON: 3 BUILD_PYTHONLESS: 1 @@ -37,43 +38,43 @@ parameters: PY3.5_90: DESIRED_PYTHON: 3.5 CUDA_VERSION: 90 - PY3.6_90: - DESIRED_PYTHON: 3.6 - CUDA_VERSION: 90 - PY3.7_90: - DESIRED_PYTHON: 3.7 - CUDA_VERSION: 90 - PY3.5_80: - DESIRED_PYTHON: 3.5 - CUDA_VERSION: 80 - PY3.6_80: - DESIRED_PYTHON: 3.6 - CUDA_VERSION: 80 - PY3.7_80: - DESIRED_PYTHON: 3.7 - CUDA_VERSION: 80 - PY3.5_100: - DESIRED_PYTHON: 3.5 - CUDA_VERSION: 100 - PY3.6_100: - DESIRED_PYTHON: 3.6 - CUDA_VERSION: 100 - PY3.7_100: - DESIRED_PYTHON: 3.7 - CUDA_VERSION: 100 + # PY3.6_90: + # DESIRED_PYTHON: 3.6 + # CUDA_VERSION: 90 + # PY3.7_90: + # DESIRED_PYTHON: 3.7 + # CUDA_VERSION: 90 + # PY3.5_80: + # DESIRED_PYTHON: 3.5 + # CUDA_VERSION: 80 + # PY3.6_80: + # DESIRED_PYTHON: 3.6 + # CUDA_VERSION: 80 + # PY3.7_80: + # DESIRED_PYTHON: 3.7 + # CUDA_VERSION: 80 + # PY3.5_100: + # DESIRED_PYTHON: 3.5 + # CUDA_VERSION: 100 + # PY3.6_100: + # DESIRED_PYTHON: 3.6 + # CUDA_VERSION: 100 + # PY3.7_100: + # DESIRED_PYTHON: 3.7 + # CUDA_VERSION: 100 ${{ if and(neq(parameters.spec, 'cpu'), eq(parameters.package, 'wheels')) }}: - LIBTORCH_80: - DESIRED_PYTHON: 3 - BUILD_PYTHONLESS: 1 - CUDA_VERSION: 80 + # LIBTORCH_80: + # DESIRED_PYTHON: 3 + # BUILD_PYTHONLESS: 1 + # CUDA_VERSION: 80 LIBTORCH_90: DESIRED_PYTHON: 3 BUILD_PYTHONLESS: 1 CUDA_VERSION: 90 - LIBTORCH_100: - DESIRED_PYTHON: 3 - BUILD_PYTHONLESS: 1 - CUDA_VERSION: 100 + # LIBTORCH_100: + # DESIRED_PYTHON: 3 + # BUILD_PYTHONLESS: 1 + # CUDA_VERSION: 100 pool: ${{ if eq(parameters.msagent, 'true') }}: From f34eaa8c8e52b5b711311006ac3526bb9137d28f Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 15:01:14 +0800 Subject: [PATCH 04/30] More fixes --- windows/azure-pipelines.yml | 62 ++++++++++++++++---------------- windows/templates/build_task.yml | 1 + 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/windows/azure-pipelines.yml b/windows/azure-pipelines.yml index e6e795e5e..fdd375b85 100644 --- a/windows/azure-pipelines.yml +++ b/windows/azure-pipelines.yml @@ -20,42 +20,42 @@ jobs: - template: templates/auth_task.yml - template: templates/build_task.yml - parameters: - package: 'wheels' - spec: 'cpu' - jobDesc: 'Wheels' - packageDesc: 'CPU' - jobEnabled: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) - msagent: $(ms.hosted.agent.cpu) + parameters: + package: 'wheels' + spec: 'cpu' + jobDesc: 'Wheels' + packageDesc: 'CPU' + jobEnabled: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) + msagent: $(ms.hosted.agent.cpu) - template: templates/build_task.yml - parameters: - package: 'conda' - spec: 'cpu' - jobDesc: 'Conda' - packageDesc: 'CPU' - jobEnabled: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) - msagent: $(ms.hosted.agent.cpu) + parameters: + package: 'conda' + spec: 'cpu' + jobDesc: 'Conda' + packageDesc: 'CPU' + jobEnabled: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) + msagent: $(ms.hosted.agent.cpu) - template: templates/build_task.yml - parameters: - package: 'wheels' - spec: 'cuda' - jobDesc: 'Wheels' - packageDesc: 'CUDA' - jobEnabled: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) - msagent: $(ms.hosted.agent.cuda) + parameters: + package: 'wheels' + spec: 'cuda' + jobDesc: 'Wheels' + packageDesc: 'CUDA' + jobEnabled: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) + msagent: $(ms.hosted.agent.cuda) - template: templates/build_task.yml - parameters: - package: 'conda' - spec: 'cuda' - jobDesc: 'Conda' - packageDesc: 'CUDA' - jobEnabled: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) - msagent: $(ms.hosted.agent.cuda) + parameters: + package: 'conda' + spec: 'cuda' + jobDesc: 'Conda' + packageDesc: 'CUDA' + jobEnabled: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) + msagent: $(ms.hosted.agent.cuda) - template: templates/linux_build_task.yml - parameters: - jobEnabled: and(and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.linux.enabled'], 'true')), eq(variables['build.conda.enabled'], 'true')) - msagent: $(ms.hosted.agent.cpu) + parameters: + jobEnabled: and(and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.linux.enabled'], 'true')), eq(variables['build.conda.enabled'], 'true')) + msagent: $(ms.hosted.agent.cpu) diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml index e112fa7cf..a5890b1f7 100644 --- a/windows/templates/build_task.yml +++ b/windows/templates/build_task.yml @@ -6,6 +6,7 @@ parameters: jobEnabled: '' msagent: '' +jobs: - job: 'Windows_${{ parameters.jobDesc }}_${{ parameters.packageDesc }}_Build' timeoutInMinutes: 0 cancelTimeoutInMinutes: 5 From 3cfc1e25e1b4ee442e37d346d912e3ad418aa4cb Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 15:02:04 +0800 Subject: [PATCH 05/30] Fixes on linux build task --- windows/templates/linux_build_task.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/windows/templates/linux_build_task.yml b/windows/templates/linux_build_task.yml index 97d038f46..febb1410a 100644 --- a/windows/templates/linux_build_task.yml +++ b/windows/templates/linux_build_task.yml @@ -2,6 +2,7 @@ parameters: jobEnabled: '' msagent: '' +jobs: - job: 'Linux_CPU_Conda_Build' timeoutInMinutes: 0 cancelTimeoutInMinutes: 5 From 551d8394fe6cd55ed2fb991ff706c5f9eb1a13dc Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 15:13:09 +0800 Subject: [PATCH 06/30] Use simple if syntax --- windows/templates/build_task.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml index a5890b1f7..b77a14581 100644 --- a/windows/templates/build_task.yml +++ b/windows/templates/build_task.yml @@ -14,8 +14,12 @@ jobs: variables: - name: CUDA_VERSION value: ${{ parameters.cudaVer }} - - name: USE_SCCACHE - value: ${{ eq(parameters.msagent, 'false') - 0 }} + - ${{ if eq(parameters.msagent, 'true') }}: + - name: USE_SCCACHE + value: 1 + - ${{ if eq(parameters.msagent, 'false') }}: + - name: USE_SCCACHE + value: 0 - ${{ if eq(parameters.package, 'conda') }}: - group: peterjc_anaconda_token - name: PYTORCH_FINAL_PACKAGE_DIR From f51e99d5ec46f939c53d2a69d18e66458fd2e883 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 15:13:56 +0800 Subject: [PATCH 07/30] Fix wrong values --- windows/templates/build_task.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml index b77a14581..329c46ced 100644 --- a/windows/templates/build_task.yml +++ b/windows/templates/build_task.yml @@ -16,10 +16,10 @@ jobs: value: ${{ parameters.cudaVer }} - ${{ if eq(parameters.msagent, 'true') }}: - name: USE_SCCACHE - value: 1 + value: 0 - ${{ if eq(parameters.msagent, 'false') }}: - name: USE_SCCACHE - value: 0 + value: 1 - ${{ if eq(parameters.package, 'conda') }}: - group: peterjc_anaconda_token - name: PYTORCH_FINAL_PACKAGE_DIR From eb5fe22db2a29fdd90702647c85725594499368d Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 15:18:23 +0800 Subject: [PATCH 08/30] Change neq to ne --- windows/templates/build_task.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml index 329c46ced..c23720b3d 100644 --- a/windows/templates/build_task.yml +++ b/windows/templates/build_task.yml @@ -39,7 +39,7 @@ jobs: LIBTORCH: DESIRED_PYTHON: 3 BUILD_PYTHONLESS: 1 - ${{ if neq(parameters.spec, 'cpu') }}: + ${{ if ne(parameters.spec, 'cpu') }}: PY3.5_90: DESIRED_PYTHON: 3.5 CUDA_VERSION: 90 @@ -67,7 +67,7 @@ jobs: # PY3.7_100: # DESIRED_PYTHON: 3.7 # CUDA_VERSION: 100 - ${{ if and(neq(parameters.spec, 'cpu'), eq(parameters.package, 'wheels')) }}: + ${{ if and(ne(parameters.spec, 'cpu'), eq(parameters.package, 'wheels')) }}: # LIBTORCH_80: # DESIRED_PYTHON: 3 # BUILD_PYTHONLESS: 1 From 0d7067ccb2530dbf021fe1bd7ec12131db1637c9 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 15:20:50 +0800 Subject: [PATCH 09/30] Change always to a normal expression --- windows/templates/build_task.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml index c23720b3d..96d8ee782 100644 --- a/windows/templates/build_task.yml +++ b/windows/templates/build_task.yml @@ -107,7 +107,7 @@ jobs: parameters: msagent: ${{ parameters.msagent }} -- ${{ if always() }}: +- ${{ if or(eq(parameters.package, 'wheels'), eq(parameters.package, 'conda')) }}: - template: publish_test_results.yml parameters: msagent: ${{ parameters.msagent }} From da0835f957db98177b2e16bb95d476cd511043ee Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 15:24:58 +0800 Subject: [PATCH 10/30] Fix indentation --- windows/templates/build_conda.yml | 2 +- windows/templates/build_wheels.yml | 2 +- windows/templates/publish_test_results.yml | 12 ++++++------ windows/templates/upload_to_conda.yml | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/windows/templates/build_conda.yml b/windows/templates/build_conda.yml index e59ffee09..5f888ca9c 100644 --- a/windows/templates/build_conda.yml +++ b/windows/templates/build_conda.yml @@ -1,5 +1,5 @@ parameters: - msagent: false + msagent: false steps: - script: 'call windows/build_pytorch.bat %CUDA_VERSION% %PYTORCH_BUILD_VERSION% %PYTORCH_BUILD_NUMBER%' diff --git a/windows/templates/build_wheels.yml b/windows/templates/build_wheels.yml index 4aa7e8403..97a5762a7 100644 --- a/windows/templates/build_wheels.yml +++ b/windows/templates/build_wheels.yml @@ -1,5 +1,5 @@ parameters: - msagent: false + msagent: false steps: - script: 'if not exist %PYTORCH_FINAL_PACKAGE_DIR% mkdir %PYTORCH_FINAL_PACKAGE_DIR%' diff --git a/windows/templates/publish_test_results.yml b/windows/templates/publish_test_results.yml index f3e89cf95..7139bda06 100644 --- a/windows/templates/publish_test_results.yml +++ b/windows/templates/publish_test_results.yml @@ -1,6 +1,6 @@ - steps: - - task: PublishTestResults@2 # No test results to publish - inputs: - testResultsFiles: 'windows/pytorch/test/**/*.xml' - testRunTitle: 'Publish test results' - enabled: false \ No newline at end of file +steps: + - task: PublishTestResults@2 # No test results to publish + inputs: + testResultsFiles: 'windows/pytorch/test/**/*.xml' + testRunTitle: 'Publish test results' + enabled: false diff --git a/windows/templates/upload_to_conda.yml b/windows/templates/upload_to_conda.yml index 3ea43a556..cc3cccaf7 100644 --- a/windows/templates/upload_to_conda.yml +++ b/windows/templates/upload_to_conda.yml @@ -7,4 +7,4 @@ steps: displayName: 'Upload packages to Anaconda Cloud' env: PYTORCH_ANACONDA_USERNAME: ${{ parameters.user }} - PYTORCH_ANACONDA_PASSWORD: ${{ parameters.pass }} \ No newline at end of file + PYTORCH_ANACONDA_PASSWORD: ${{ parameters.pass }} From e723a4093d986c62eeef4e956f5702c203691d09 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 15:35:56 +0800 Subject: [PATCH 11/30] Fix more indentions --- windows/templates/build_conda.yml | 10 ++--- windows/templates/build_wheels.yml | 14 +++---- windows/templates/publish_test_results.yml | 10 ++--- windows/templates/setup_nightly_variables.yml | 10 ++--- windows/templates/upload_to_conda.yml | 10 ++--- windows/templates/upload_to_s3.yml | 40 +++++++++---------- 6 files changed, 47 insertions(+), 47 deletions(-) diff --git a/windows/templates/build_conda.yml b/windows/templates/build_conda.yml index 5f888ca9c..e27ba44ee 100644 --- a/windows/templates/build_conda.yml +++ b/windows/templates/build_conda.yml @@ -2,8 +2,8 @@ parameters: msagent: false steps: - - script: 'call windows/build_pytorch.bat %CUDA_VERSION% %PYTORCH_BUILD_VERSION% %PYTORCH_BUILD_NUMBER%' - displayName: Build - env: - ${{ if eq(parameters.msagent, 'true') }}: - MAX_JOBS: 2 +- script: 'call windows/build_pytorch.bat %CUDA_VERSION% %PYTORCH_BUILD_VERSION% %PYTORCH_BUILD_NUMBER%' + displayName: Build + env: + ${{ if eq(parameters.msagent, 'true') }}: + MAX_JOBS: 2 diff --git a/windows/templates/build_wheels.yml b/windows/templates/build_wheels.yml index 97a5762a7..ebf9672fa 100644 --- a/windows/templates/build_wheels.yml +++ b/windows/templates/build_wheels.yml @@ -2,11 +2,11 @@ parameters: msagent: false steps: - - script: 'if not exist %PYTORCH_FINAL_PACKAGE_DIR% mkdir %PYTORCH_FINAL_PACKAGE_DIR%' - displayName: 'Create final package directory' +- script: 'if not exist %PYTORCH_FINAL_PACKAGE_DIR% mkdir %PYTORCH_FINAL_PACKAGE_DIR%' + displayName: 'Create final package directory' - - script: 'call windows/build_pytorch.bat %CUDA_VERSION% %PYTORCH_BUILD_VERSION% %PYTORCH_BUILD_NUMBER%' - displayName: Build - env: - ${{ if eq(parameters.msagent, 'true') }}: - MAX_JOBS: 2 +- script: 'call windows/build_pytorch.bat %CUDA_VERSION% %PYTORCH_BUILD_VERSION% %PYTORCH_BUILD_NUMBER%' + displayName: Build + env: + ${{ if eq(parameters.msagent, 'true') }}: + MAX_JOBS: 2 diff --git a/windows/templates/publish_test_results.yml b/windows/templates/publish_test_results.yml index 7139bda06..1e0dc0215 100644 --- a/windows/templates/publish_test_results.yml +++ b/windows/templates/publish_test_results.yml @@ -1,6 +1,6 @@ steps: - - task: PublishTestResults@2 # No test results to publish - inputs: - testResultsFiles: 'windows/pytorch/test/**/*.xml' - testRunTitle: 'Publish test results' - enabled: false +- task: PublishTestResults@2 # No test results to publish + inputs: + testResultsFiles: 'windows/pytorch/test/**/*.xml' + testRunTitle: 'Publish test results' + enabled: false diff --git a/windows/templates/setup_nightly_variables.yml b/windows/templates/setup_nightly_variables.yml index 24f8cd2fc..d77da155f 100644 --- a/windows/templates/setup_nightly_variables.yml +++ b/windows/templates/setup_nightly_variables.yml @@ -1,7 +1,7 @@ steps: - - task: BatchScript@1 - displayName: 'Setup nightly variables' - inputs: - filename: 'windows/internal/nightly_defaults.bat' +- task: BatchScript@1 + displayName: 'Setup nightly variables' + inputs: + filename: 'windows/internal/nightly_defaults.bat' - modifyEnvironment: true + modifyEnvironment: true diff --git a/windows/templates/upload_to_conda.yml b/windows/templates/upload_to_conda.yml index cc3cccaf7..2da208733 100644 --- a/windows/templates/upload_to_conda.yml +++ b/windows/templates/upload_to_conda.yml @@ -3,8 +3,8 @@ parameters: pass: '' steps: - - script: 'call windows/internal/upload.bat' - displayName: 'Upload packages to Anaconda Cloud' - env: - PYTORCH_ANACONDA_USERNAME: ${{ parameters.user }} - PYTORCH_ANACONDA_PASSWORD: ${{ parameters.pass }} +- script: 'call windows/internal/upload.bat' + displayName: 'Upload packages to Anaconda Cloud' + env: + PYTORCH_ANACONDA_USERNAME: ${{ parameters.user }} + PYTORCH_ANACONDA_PASSWORD: ${{ parameters.pass }} diff --git a/windows/templates/upload_to_s3.yml b/windows/templates/upload_to_s3.yml index b2a80c55d..8333a5834 100644 --- a/windows/templates/upload_to_s3.yml +++ b/windows/templates/upload_to_s3.yml @@ -3,24 +3,24 @@ parameters: cudaVer: '' steps: - - task: AmazonWebServices.aws-vsts-tools.S3Upload.S3Upload@1 - displayName: 'Upload ${{ parameters.cuVer }} wheel to S3' - inputs: - awsCredentials: 'Pytorch S3 bucket' - bucketName: 'pytorch' - sourceFolder: 'windows/output/${{ parameters.cudaVer }}' - globExpressions: '*.whl' - targetFolder: 'whl/nightly/${{ parameters.cuVer }}/' - filesAcl: 'public-read' - flattenFolders: 'true' +- task: AmazonWebServices.aws-vsts-tools.S3Upload.S3Upload@1 + displayName: 'Upload ${{ parameters.cuVer }} wheel to S3' + inputs: + awsCredentials: 'Pytorch S3 bucket' + bucketName: 'pytorch' + sourceFolder: 'windows/output/${{ parameters.cudaVer }}' + globExpressions: '*.whl' + targetFolder: 'whl/nightly/${{ parameters.cuVer }}/' + filesAcl: 'public-read' + flattenFolders: 'true' - - task: AmazonWebServices.aws-vsts-tools.S3Upload.S3Upload@1 - displayName: 'Upload ${{ parameters.cuVer }} libtorch to S3' - inputs: - awsCredentials: 'Pytorch S3 bucket' - bucketName: 'pytorch' - sourceFolder: 'windows/output/${{ parameters.cudaVer }}' - globExpressions: '*.zip' - targetFolder: 'libtorch/nightly/${{ parameters.cuVer }}/' - filesAcl: 'public-read' - flattenFolders: 'true' +- task: AmazonWebServices.aws-vsts-tools.S3Upload.S3Upload@1 + displayName: 'Upload ${{ parameters.cuVer }} libtorch to S3' + inputs: + awsCredentials: 'Pytorch S3 bucket' + bucketName: 'pytorch' + sourceFolder: 'windows/output/${{ parameters.cudaVer }}' + globExpressions: '*.zip' + targetFolder: 'libtorch/nightly/${{ parameters.cuVer }}/' + filesAcl: 'public-read' + flattenFolders: 'true' From 8cb3a582407422a3075766866cbc87320009b1bd Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 15:41:30 +0800 Subject: [PATCH 12/30] . --- windows/templates/build_wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/windows/templates/build_wheels.yml b/windows/templates/build_wheels.yml index ebf9672fa..909e6ef2e 100644 --- a/windows/templates/build_wheels.yml +++ b/windows/templates/build_wheels.yml @@ -1,5 +1,5 @@ -parameters: - msagent: false +# parameters: +# msagent: false steps: - script: 'if not exist %PYTORCH_FINAL_PACKAGE_DIR% mkdir %PYTORCH_FINAL_PACKAGE_DIR%' From e71c651a40304e4379e9c550370f2512680436d8 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 15:42:08 +0800 Subject: [PATCH 13/30] . --- windows/templates/build_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows/templates/build_wheels.yml b/windows/templates/build_wheels.yml index 909e6ef2e..dc9e7b7a3 100644 --- a/windows/templates/build_wheels.yml +++ b/windows/templates/build_wheels.yml @@ -1,7 +1,7 @@ # parameters: # msagent: false -steps: +# steps: - script: 'if not exist %PYTORCH_FINAL_PACKAGE_DIR% mkdir %PYTORCH_FINAL_PACKAGE_DIR%' displayName: 'Create final package directory' From a1e2ec26537b6f6097f3afd7f9003ba8da2b06d6 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 15:43:49 +0800 Subject: [PATCH 14/30] Fix indention again --- windows/templates/build_task.yml | 44 +++++++++++++++--------------- windows/templates/build_wheels.yml | 6 ++-- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml index 96d8ee782..46b1acca1 100644 --- a/windows/templates/build_task.yml +++ b/windows/templates/build_task.yml @@ -97,29 +97,29 @@ jobs: - template: setup_nightly_variables.yml -- ${{ if eq(parameters.package, 'wheels') }}: - - template: build_wheels.yml - parameters: - msagent: ${{ parameters.msagent }} + - ${{ if eq(parameters.package, 'wheels') }}: + - template: build_wheels.yml + parameters: + msagent: ${{ parameters.msagent }} -- ${{ if eq(parameters.package, 'conda') }}: - - template: build_conda.yml - parameters: - msagent: ${{ parameters.msagent }} + - ${{ if eq(parameters.package, 'conda') }}: + - template: build_conda.yml + parameters: + msagent: ${{ parameters.msagent }} -- ${{ if or(eq(parameters.package, 'wheels'), eq(parameters.package, 'conda')) }}: - - template: publish_test_results.yml - parameters: - msagent: ${{ parameters.msagent }} + - ${{ if or(eq(parameters.package, 'wheels'), eq(parameters.package, 'conda')) }}: + - template: publish_test_results.yml + parameters: + msagent: ${{ parameters.msagent }} -- ${{ if eq(parameters.package, 'wheels') }}: - - template: upload_to_s3.yml - parameters: - cuVer: 'cu$(CUDA_VERSION)' - cudaVer: 'cuda$(CUDA_VERSION)' + - ${{ if eq(parameters.package, 'wheels') }}: + - template: upload_to_s3.yml + parameters: + cuVer: 'cu$(CUDA_VERSION)' + cudaVer: 'cuda$(CUDA_VERSION)' -- ${{ if eq(parameters.package, 'conda') }}: - - template: upload_to_conda.yml - parameters: - user: $(peterjc_conda_username) - pass: $(peterjc_conda_password) + - ${{ if eq(parameters.package, 'conda') }}: + - template: upload_to_conda.yml + parameters: + user: $(peterjc_conda_username) + pass: $(peterjc_conda_password) diff --git a/windows/templates/build_wheels.yml b/windows/templates/build_wheels.yml index dc9e7b7a3..ebf9672fa 100644 --- a/windows/templates/build_wheels.yml +++ b/windows/templates/build_wheels.yml @@ -1,7 +1,7 @@ -# parameters: -# msagent: false +parameters: + msagent: false -# steps: +steps: - script: 'if not exist %PYTORCH_FINAL_PACKAGE_DIR% mkdir %PYTORCH_FINAL_PACKAGE_DIR%' displayName: 'Create final package directory' From d828e1adc61a2a5fc22b260fa4997ecf67af8601 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 15:48:59 +0800 Subject: [PATCH 15/30] . --- windows/templates/build_task.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml index 46b1acca1..1d0425755 100644 --- a/windows/templates/build_task.yml +++ b/windows/templates/build_task.yml @@ -103,7 +103,7 @@ jobs: msagent: ${{ parameters.msagent }} - ${{ if eq(parameters.package, 'conda') }}: - - template: build_conda.yml + - template: ..\build_conda.yml parameters: msagent: ${{ parameters.msagent }} From 80872a27c82075e00c3fbe1f1002d3e6e3a794bd Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 15:55:18 +0800 Subject: [PATCH 16/30] Use absolute path --- windows/templates/build_task.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml index 1d0425755..721409811 100644 --- a/windows/templates/build_task.yml +++ b/windows/templates/build_task.yml @@ -91,35 +91,35 @@ jobs: - checkout: self clean: true - - template: setup_env_for_msagent.yml + - template: '$(Build.Repository.LocalPath)\windows\templates\setup_env_for_msagent.yml' parameters: msagent: ${{ parameters.msagent }} - template: setup_nightly_variables.yml - ${{ if eq(parameters.package, 'wheels') }}: - - template: build_wheels.yml + - template: '$(Build.Repository.LocalPath)\windows\templates\build_wheels.yml' parameters: msagent: ${{ parameters.msagent }} - ${{ if eq(parameters.package, 'conda') }}: - - template: ..\build_conda.yml + - template: '$(Build.Repository.LocalPath)\windows\templates\build_conda.yml' parameters: msagent: ${{ parameters.msagent }} - ${{ if or(eq(parameters.package, 'wheels'), eq(parameters.package, 'conda')) }}: - - template: publish_test_results.yml + - template: '$(Build.Repository.LocalPath)\windows\templates\publish_test_results.yml' parameters: msagent: ${{ parameters.msagent }} - ${{ if eq(parameters.package, 'wheels') }}: - - template: upload_to_s3.yml + - template: '$(Build.Repository.LocalPath)\windows\templates\upload_to_s3.yml' parameters: cuVer: 'cu$(CUDA_VERSION)' cudaVer: 'cuda$(CUDA_VERSION)' - ${{ if eq(parameters.package, 'conda') }}: - - template: upload_to_conda.yml + - template: '$(Build.Repository.LocalPath)\windows\templates\upload_to_conda.yml' parameters: user: $(peterjc_conda_username) pass: $(peterjc_conda_password) From a6116d79f0040db2563873cf66e08574789a6230 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 15:56:24 +0800 Subject: [PATCH 17/30] . --- windows/templates/build_task.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml index 721409811..1472334a9 100644 --- a/windows/templates/build_task.yml +++ b/windows/templates/build_task.yml @@ -91,35 +91,35 @@ jobs: - checkout: self clean: true - - template: '$(Build.Repository.LocalPath)\windows\templates\setup_env_for_msagent.yml' + - template: '\windows\templates\setup_env_for_msagent.yml' parameters: msagent: ${{ parameters.msagent }} - template: setup_nightly_variables.yml - ${{ if eq(parameters.package, 'wheels') }}: - - template: '$(Build.Repository.LocalPath)\windows\templates\build_wheels.yml' + - template: '\windows\templates\build_wheels.yml' parameters: msagent: ${{ parameters.msagent }} - ${{ if eq(parameters.package, 'conda') }}: - - template: '$(Build.Repository.LocalPath)\windows\templates\build_conda.yml' + - template: '\windows\templates\build_conda.yml' parameters: msagent: ${{ parameters.msagent }} - ${{ if or(eq(parameters.package, 'wheels'), eq(parameters.package, 'conda')) }}: - - template: '$(Build.Repository.LocalPath)\windows\templates\publish_test_results.yml' + - template: '\windows\templates\publish_test_results.yml' parameters: msagent: ${{ parameters.msagent }} - ${{ if eq(parameters.package, 'wheels') }}: - - template: '$(Build.Repository.LocalPath)\windows\templates\upload_to_s3.yml' + - template: '\windows\templates\upload_to_s3.yml' parameters: cuVer: 'cu$(CUDA_VERSION)' cudaVer: 'cuda$(CUDA_VERSION)' - ${{ if eq(parameters.package, 'conda') }}: - - template: '$(Build.Repository.LocalPath)\windows\templates\upload_to_conda.yml' + - template: '\windows\templates\upload_to_conda.yml' parameters: user: $(peterjc_conda_username) pass: $(peterjc_conda_password) From 2a46568acabe23a2962c9f16a046142cba64f6a5 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 15:58:38 +0800 Subject: [PATCH 18/30] Fix path issues --- windows/templates/build_task.yml | 12 ++++++------ windows/templates/linux_build_task.yml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml index 1472334a9..46b1acca1 100644 --- a/windows/templates/build_task.yml +++ b/windows/templates/build_task.yml @@ -91,35 +91,35 @@ jobs: - checkout: self clean: true - - template: '\windows\templates\setup_env_for_msagent.yml' + - template: setup_env_for_msagent.yml parameters: msagent: ${{ parameters.msagent }} - template: setup_nightly_variables.yml - ${{ if eq(parameters.package, 'wheels') }}: - - template: '\windows\templates\build_wheels.yml' + - template: build_wheels.yml parameters: msagent: ${{ parameters.msagent }} - ${{ if eq(parameters.package, 'conda') }}: - - template: '\windows\templates\build_conda.yml' + - template: build_conda.yml parameters: msagent: ${{ parameters.msagent }} - ${{ if or(eq(parameters.package, 'wheels'), eq(parameters.package, 'conda')) }}: - - template: '\windows\templates\publish_test_results.yml' + - template: publish_test_results.yml parameters: msagent: ${{ parameters.msagent }} - ${{ if eq(parameters.package, 'wheels') }}: - - template: '\windows\templates\upload_to_s3.yml' + - template: upload_to_s3.yml parameters: cuVer: 'cu$(CUDA_VERSION)' cudaVer: 'cuda$(CUDA_VERSION)' - ${{ if eq(parameters.package, 'conda') }}: - - template: '\windows\templates\upload_to_conda.yml' + - template: upload_to_conda.yml parameters: user: $(peterjc_conda_username) pass: $(peterjc_conda_password) diff --git a/windows/templates/linux_build_task.yml b/windows/templates/linux_build_task.yml index febb1410a..9453fe4f1 100644 --- a/windows/templates/linux_build_task.yml +++ b/windows/templates/linux_build_task.yml @@ -33,6 +33,6 @@ jobs: inputs: packageSpecs: 'conda-build' - - template: templates/build_conda.yml + - template: build_conda.yml parameters: msagent: ${{ parameters.msagent }} From b2b84f8f4f98bd61d4c659b9efd697dc67e927dc Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 16:32:09 +0800 Subject: [PATCH 19/30] Fix if conditions --- windows/azure-pipelines.yml | 26 +++++++++++---------- windows/templates/build_conda.yml | 7 ++++-- windows/templates/build_task.yml | 16 ++++++------- windows/templates/build_wheels.yml | 5 +--- windows/templates/linux_build_task.yml | 2 -- windows/templates/setup_env_for_msagent.yml | 2 +- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/windows/azure-pipelines.yml b/windows/azure-pipelines.yml index fdd375b85..6b6bb9bec 100644 --- a/windows/azure-pipelines.yml +++ b/windows/azure-pipelines.yml @@ -8,54 +8,56 @@ variables: # pytorch.build.number: 1 # pytorch.branch: master # pytorch.repo: pytorch - build.linux.enabled: false - build.cpu.enabled: true - build.cuda.enabled: true - build.wheels.enabled: true - build.conda.enabled: true - ms.hosted.agent.cpu: true - ms.hosted.agent.cuda: false + build.linux.enabled: False + build.cpu.enabled: True + build.cuda.enabled: True + build.wheels.enabled: True + build.conda.enabled: True + ms.hosted.agent.cpu: True + ms.hosted.agent.cuda: False jobs: - template: templates/auth_task.yml + - template: templates/build_task.yml + condition: and(variables['build.cpu.enabled'], variables['build.wheels.enabled']) parameters: package: 'wheels' spec: 'cpu' jobDesc: 'Wheels' packageDesc: 'CPU' - jobEnabled: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) msagent: $(ms.hosted.agent.cpu) + - template: templates/build_task.yml + jobEnabled: and(variables['build.cpu.enabled'], variables['build.conda.enabled']) parameters: package: 'conda' spec: 'cpu' jobDesc: 'Conda' packageDesc: 'CPU' - jobEnabled: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) msagent: $(ms.hosted.agent.cpu) - template: templates/build_task.yml + jobEnabled: and(variables['build.cuda.enabled'], variables['build.wheels.enabled']) parameters: package: 'wheels' spec: 'cuda' jobDesc: 'Wheels' packageDesc: 'CUDA' - jobEnabled: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) msagent: $(ms.hosted.agent.cuda) - template: templates/build_task.yml + jobEnabled: and(variables['build.cuda.enabled'], variables['build.conda.enabled']) parameters: package: 'conda' spec: 'cuda' jobDesc: 'Conda' packageDesc: 'CUDA' - jobEnabled: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) msagent: $(ms.hosted.agent.cuda) - template: templates/linux_build_task.yml + condition: and(variables['build.cpu.enabled'], variables['build.linux.enabled'], variables['build.conda.enabled']) parameters: - jobEnabled: and(and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.linux.enabled'], 'true')), eq(variables['build.conda.enabled'], 'true')) msagent: $(ms.hosted.agent.cpu) diff --git a/windows/templates/build_conda.yml b/windows/templates/build_conda.yml index e27ba44ee..7b228c8e0 100644 --- a/windows/templates/build_conda.yml +++ b/windows/templates/build_conda.yml @@ -2,8 +2,11 @@ parameters: msagent: false steps: -- script: 'call windows/build_pytorch.bat %CUDA_VERSION% %PYTORCH_BUILD_VERSION% %PYTORCH_BUILD_NUMBER%' +- script: 'if not exist %PYTORCH_FINAL_PACKAGE_DIR% mkdir %PYTORCH_FINAL_PACKAGE_DIR%' + displayName: 'Create final package directory' + +- bash: './conda/build_pytorch.sh $CUDA_VERSION $PYTORCH_BUILD_VERSION $PYTORCH_BUILD_NUMBER' displayName: Build env: - ${{ if eq(parameters.msagent, 'true') }}: + ${{ if parameters.msagent }}: MAX_JOBS: 2 diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml index 46b1acca1..01f250b0e 100644 --- a/windows/templates/build_task.yml +++ b/windows/templates/build_task.yml @@ -3,21 +3,17 @@ parameters: spec: '' jobDesc: '' packageDesc: '' - jobEnabled: '' msagent: '' jobs: - job: 'Windows_${{ parameters.jobDesc }}_${{ parameters.packageDesc }}_Build' timeoutInMinutes: 0 cancelTimeoutInMinutes: 5 - condition: ${{ and(parameters.jobEnabled, 'true') }} variables: - - name: CUDA_VERSION - value: ${{ parameters.cudaVer }} - - ${{ if eq(parameters.msagent, 'true') }}: + - ${{ if parameters.msagent }}: - name: USE_SCCACHE value: 0 - - ${{ if eq(parameters.msagent, 'false') }}: + - ${{ if not(parameters.msagent) }}: - name: USE_SCCACHE value: 1 - ${{ if eq(parameters.package, 'conda') }}: @@ -31,13 +27,17 @@ jobs: ${{ if eq(parameters.spec, 'cpu') }}: PY3.5: DESIRED_PYTHON: 3.5 + CUDA_VERSION: cpu # PY3.6: # DESIRED_PYTHON: 3.6 + # CUDA_VERSION: cpu # PY3.7: # DESIRED_PYTHON: 3.7 + # CUDA_VERSION: cpu ${{ if and(eq(parameters.package, 'wheels'), eq(parameters.spec, 'cpu')) }}: LIBTORCH: DESIRED_PYTHON: 3 + CUDA_VERSION: cpu BUILD_PYTHONLESS: 1 ${{ if ne(parameters.spec, 'cpu') }}: PY3.5_90: @@ -82,9 +82,9 @@ jobs: # CUDA_VERSION: 100 pool: - ${{ if eq(parameters.msagent, 'true') }}: + ${{ if parameters.msagent }}: vmImage: 'win1803' - ${{ if eq(parameters.msagent, 'false') }}: + ${{ if not(parameters.msagent) }}: name: 'pytorch' steps: diff --git a/windows/templates/build_wheels.yml b/windows/templates/build_wheels.yml index ebf9672fa..151a7c871 100644 --- a/windows/templates/build_wheels.yml +++ b/windows/templates/build_wheels.yml @@ -2,11 +2,8 @@ parameters: msagent: false steps: -- script: 'if not exist %PYTORCH_FINAL_PACKAGE_DIR% mkdir %PYTORCH_FINAL_PACKAGE_DIR%' - displayName: 'Create final package directory' - - script: 'call windows/build_pytorch.bat %CUDA_VERSION% %PYTORCH_BUILD_VERSION% %PYTORCH_BUILD_NUMBER%' displayName: Build env: - ${{ if eq(parameters.msagent, 'true') }}: + ${{ if parameters.msagent }}: MAX_JOBS: 2 diff --git a/windows/templates/linux_build_task.yml b/windows/templates/linux_build_task.yml index 9453fe4f1..b85a63a0c 100644 --- a/windows/templates/linux_build_task.yml +++ b/windows/templates/linux_build_task.yml @@ -1,12 +1,10 @@ parameters: - jobEnabled: '' msagent: '' jobs: - job: 'Linux_CPU_Conda_Build' timeoutInMinutes: 0 cancelTimeoutInMinutes: 5 - condition: ${{ and(parameters.jobEnabled, 'true') }} variables: CUDA_VERSION: cpu TORCH_CONDA_BUILD_FOLDER: pytorch-nightly diff --git a/windows/templates/setup_env_for_msagent.yml b/windows/templates/setup_env_for_msagent.yml index a8d35692b..b87e2da03 100644 --- a/windows/templates/setup_env_for_msagent.yml +++ b/windows/templates/setup_env_for_msagent.yml @@ -2,7 +2,7 @@ parameters: msagent: false steps: -- ${{ if eq(parameters.msagent, 'true') }}: +- ${{ if parameters.msagent }}: - task: BatchScript@1 displayName: 'Install VS 2017' inputs: From 6cd4efb91150e25d8542a7440a10d5317fa07f98 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 17:07:14 +0800 Subject: [PATCH 20/30] Revert conditions --- windows/azure-pipelines.yml | 22 +++++++++++++++++++--- windows/templates/build_task.yml | 9 +++++++++ windows/templates/linux_build_task.yml | 4 ++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/windows/azure-pipelines.yml b/windows/azure-pipelines.yml index 6b6bb9bec..cbfb45830 100644 --- a/windows/azure-pipelines.yml +++ b/windows/azure-pipelines.yml @@ -19,7 +19,6 @@ variables: jobs: - template: templates/auth_task.yml - - template: templates/build_task.yml condition: and(variables['build.cpu.enabled'], variables['build.wheels.enabled']) parameters: @@ -28,7 +27,10 @@ jobs: jobDesc: 'Wheels' packageDesc: 'CPU' msagent: $(ms.hosted.agent.cpu) - + cpuEnabled: $(build.cpu.enabled) + cudaEnabled: $(build.cuda.enabled) + condaEnabled: $(build.conda.enabled) + wheelsEnabled: $(build.conda.enabled) - template: templates/build_task.yml jobEnabled: and(variables['build.cpu.enabled'], variables['build.conda.enabled']) @@ -38,6 +40,10 @@ jobs: jobDesc: 'Conda' packageDesc: 'CPU' msagent: $(ms.hosted.agent.cpu) + cpuEnabled: $(build.cpu.enabled) + cudaEnabled: $(build.cuda.enabled) + condaEnabled: $(build.conda.enabled) + wheelsEnabled: $(build.conda.enabled) - template: templates/build_task.yml jobEnabled: and(variables['build.cuda.enabled'], variables['build.wheels.enabled']) @@ -47,6 +53,10 @@ jobs: jobDesc: 'Wheels' packageDesc: 'CUDA' msagent: $(ms.hosted.agent.cuda) + cpuEnabled: $(build.cpu.enabled) + cudaEnabled: $(build.cuda.enabled) + condaEnabled: $(build.conda.enabled) + wheelsEnabled: $(build.conda.enabled) - template: templates/build_task.yml jobEnabled: and(variables['build.cuda.enabled'], variables['build.conda.enabled']) @@ -56,8 +66,14 @@ jobs: jobDesc: 'Conda' packageDesc: 'CUDA' msagent: $(ms.hosted.agent.cuda) + cpuEnabled: $(build.cpu.enabled) + cudaEnabled: $(build.cuda.enabled) + condaEnabled: $(build.conda.enabled) + wheelsEnabled: $(build.conda.enabled) - template: templates/linux_build_task.yml - condition: and(variables['build.cpu.enabled'], variables['build.linux.enabled'], variables['build.conda.enabled']) parameters: msagent: $(ms.hosted.agent.cpu) + cpuEnabled: $(build.cpu.enabled) + linuxEnabled: $(build.linux.enabled) + condaEnabled: $(build.conda.enabled) diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml index 01f250b0e..cdf07df42 100644 --- a/windows/templates/build_task.yml +++ b/windows/templates/build_task.yml @@ -4,11 +4,20 @@ parameters: jobDesc: '' packageDesc: '' msagent: '' + cpuEnabled: '' + cudaEnabled: '' + condaEnabled: '' + wheelsEnabled: '' jobs: - job: 'Windows_${{ parameters.jobDesc }}_${{ parameters.packageDesc }}_Build' timeoutInMinutes: 0 cancelTimeoutInMinutes: 5 + condition: > + or(and(eq(${{ parameters.package }}, 'conda'), eq(${{ parameters.spec }}, 'cpu'), ${{ parameters.condaEnabled }}, ${{ parameters.cpuEnabled }}), + and(eq(${{ parameters.package }}, 'wheels'), eq(${{ parameters.spec }}, 'cpu'), ${{ parameters.wheelsEnabled }}, ${{ parameters.cpuEnabled }}), + and(eq(${{ parameters.package }}, 'conda'), eq(${{ parameters.spec }}, 'cuda'), ${{ parameters.condaEnabled }}, ${{ parameters.cudaEnabled }}), + and(eq(${{ parameters.package }}, 'wheels'), eq(${{ parameters.spec }}, 'cuda'), ${{ parameters.wheelsEnabled }}, ${{ parameters.cudaEnabled }})) variables: - ${{ if parameters.msagent }}: - name: USE_SCCACHE diff --git a/windows/templates/linux_build_task.yml b/windows/templates/linux_build_task.yml index b85a63a0c..e8ecb5308 100644 --- a/windows/templates/linux_build_task.yml +++ b/windows/templates/linux_build_task.yml @@ -1,10 +1,14 @@ parameters: msagent: '' + cpuEnabled: '' + linuxEnabled: '' + condaEnabled: '' jobs: - job: 'Linux_CPU_Conda_Build' timeoutInMinutes: 0 cancelTimeoutInMinutes: 5 + condition: and({{ parameters.cpuEnabled }}, {{ parameters.linuxEnabled }}, {{ parameters.condaEnabled }}) variables: CUDA_VERSION: cpu TORCH_CONDA_BUILD_FOLDER: pytorch-nightly From 17aa008c3d87de18e0656eab27aaf1f2bee51c7b Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 17:09:57 +0800 Subject: [PATCH 21/30] Remove conditions for templates --- windows/azure-pipelines.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/windows/azure-pipelines.yml b/windows/azure-pipelines.yml index cbfb45830..f50c49611 100644 --- a/windows/azure-pipelines.yml +++ b/windows/azure-pipelines.yml @@ -20,7 +20,6 @@ jobs: - template: templates/auth_task.yml - template: templates/build_task.yml - condition: and(variables['build.cpu.enabled'], variables['build.wheels.enabled']) parameters: package: 'wheels' spec: 'cpu' @@ -33,7 +32,6 @@ jobs: wheelsEnabled: $(build.conda.enabled) - template: templates/build_task.yml - jobEnabled: and(variables['build.cpu.enabled'], variables['build.conda.enabled']) parameters: package: 'conda' spec: 'cpu' @@ -46,7 +44,6 @@ jobs: wheelsEnabled: $(build.conda.enabled) - template: templates/build_task.yml - jobEnabled: and(variables['build.cuda.enabled'], variables['build.wheels.enabled']) parameters: package: 'wheels' spec: 'cuda' @@ -59,7 +56,6 @@ jobs: wheelsEnabled: $(build.conda.enabled) - template: templates/build_task.yml - jobEnabled: and(variables['build.cuda.enabled'], variables['build.conda.enabled']) parameters: package: 'conda' spec: 'cuda' From 0efb198c48c036f8318a0855d0416f7a7a2348df Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 17:13:58 +0800 Subject: [PATCH 22/30] Add quotes on strings --- windows/templates/build_task.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml index cdf07df42..a7360d359 100644 --- a/windows/templates/build_task.yml +++ b/windows/templates/build_task.yml @@ -14,10 +14,10 @@ jobs: timeoutInMinutes: 0 cancelTimeoutInMinutes: 5 condition: > - or(and(eq(${{ parameters.package }}, 'conda'), eq(${{ parameters.spec }}, 'cpu'), ${{ parameters.condaEnabled }}, ${{ parameters.cpuEnabled }}), - and(eq(${{ parameters.package }}, 'wheels'), eq(${{ parameters.spec }}, 'cpu'), ${{ parameters.wheelsEnabled }}, ${{ parameters.cpuEnabled }}), - and(eq(${{ parameters.package }}, 'conda'), eq(${{ parameters.spec }}, 'cuda'), ${{ parameters.condaEnabled }}, ${{ parameters.cudaEnabled }}), - and(eq(${{ parameters.package }}, 'wheels'), eq(${{ parameters.spec }}, 'cuda'), ${{ parameters.wheelsEnabled }}, ${{ parameters.cudaEnabled }})) + or(and(eq('${{ parameters.package }}', 'conda'), eq('${{ parameters.spec }}', 'cpu'), ${{ parameters.condaEnabled }}, ${{ parameters.cpuEnabled }}), + and(eq('${{ parameters.package }}', 'wheels'), eq('${{ parameters.spec }}', 'cpu'), ${{ parameters.wheelsEnabled }}, ${{ parameters.cpuEnabled }}), + and(eq('${{ parameters.package }}', 'conda'), eq('${{ parameters.spec }}', 'cuda'), ${{ parameters.condaEnabled }}, ${{ parameters.cudaEnabled }}), + and(eq('${{ parameters.package }}', 'wheels'), eq('${{ parameters.spec }}', 'cuda'), ${{ parameters.wheelsEnabled }}, ${{ parameters.cudaEnabled }})) variables: - ${{ if parameters.msagent }}: - name: USE_SCCACHE From a4900392613384f038de752af89ebfe685cec53a Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 17:32:28 +0800 Subject: [PATCH 23/30] . --- windows/azure-pipelines.yml | 40 +++----------------------- windows/templates/build_task.yml | 20 +++++++++---- windows/templates/linux_build_task.yml | 6 ++-- 3 files changed, 21 insertions(+), 45 deletions(-) diff --git a/windows/azure-pipelines.yml b/windows/azure-pipelines.yml index f50c49611..97fa60346 100644 --- a/windows/azure-pipelines.yml +++ b/windows/azure-pipelines.yml @@ -3,19 +3,6 @@ trigger: none pr: none -variables: - # pytorch.build.version: '1.0.0' - # pytorch.build.number: 1 - # pytorch.branch: master - # pytorch.repo: pytorch - build.linux.enabled: False - build.cpu.enabled: True - build.cuda.enabled: True - build.wheels.enabled: True - build.conda.enabled: True - ms.hosted.agent.cpu: True - ms.hosted.agent.cuda: False - jobs: - template: templates/auth_task.yml @@ -25,11 +12,7 @@ jobs: spec: 'cpu' jobDesc: 'Wheels' packageDesc: 'CPU' - msagent: $(ms.hosted.agent.cpu) - cpuEnabled: $(build.cpu.enabled) - cudaEnabled: $(build.cuda.enabled) - condaEnabled: $(build.conda.enabled) - wheelsEnabled: $(build.conda.enabled) + msagent: True - template: templates/build_task.yml parameters: @@ -37,11 +20,7 @@ jobs: spec: 'cpu' jobDesc: 'Conda' packageDesc: 'CPU' - msagent: $(ms.hosted.agent.cpu) - cpuEnabled: $(build.cpu.enabled) - cudaEnabled: $(build.cuda.enabled) - condaEnabled: $(build.conda.enabled) - wheelsEnabled: $(build.conda.enabled) + msagent: True - template: templates/build_task.yml parameters: @@ -49,11 +28,7 @@ jobs: spec: 'cuda' jobDesc: 'Wheels' packageDesc: 'CUDA' - msagent: $(ms.hosted.agent.cuda) - cpuEnabled: $(build.cpu.enabled) - cudaEnabled: $(build.cuda.enabled) - condaEnabled: $(build.conda.enabled) - wheelsEnabled: $(build.conda.enabled) + msagent: False - template: templates/build_task.yml parameters: @@ -61,15 +36,8 @@ jobs: spec: 'cuda' jobDesc: 'Conda' packageDesc: 'CUDA' - msagent: $(ms.hosted.agent.cuda) - cpuEnabled: $(build.cpu.enabled) - cudaEnabled: $(build.cuda.enabled) - condaEnabled: $(build.conda.enabled) - wheelsEnabled: $(build.conda.enabled) + msagent: False - template: templates/linux_build_task.yml parameters: msagent: $(ms.hosted.agent.cpu) - cpuEnabled: $(build.cpu.enabled) - linuxEnabled: $(build.linux.enabled) - condaEnabled: $(build.conda.enabled) diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml index a7360d359..accd53b8e 100644 --- a/windows/templates/build_task.yml +++ b/windows/templates/build_task.yml @@ -3,11 +3,12 @@ parameters: spec: '' jobDesc: '' packageDesc: '' - msagent: '' - cpuEnabled: '' - cudaEnabled: '' - condaEnabled: '' - wheelsEnabled: '' + msagent: True + cpuEnabled: True + cudaEnabled: True + condaEnabled: True + wheelsEnabled: True + override: False jobs: - job: 'Windows_${{ parameters.jobDesc }}_${{ parameters.packageDesc }}_Build' @@ -19,6 +20,15 @@ jobs: and(eq('${{ parameters.package }}', 'conda'), eq('${{ parameters.spec }}', 'cuda'), ${{ parameters.condaEnabled }}, ${{ parameters.cudaEnabled }}), and(eq('${{ parameters.package }}', 'wheels'), eq('${{ parameters.spec }}', 'cuda'), ${{ parameters.wheelsEnabled }}, ${{ parameters.cudaEnabled }})) variables: + - ${{ if parameters.override }}: + - name: PYTORCH_BUILD_VERSION + - value: '1.0.0' + - name: PYTORCH_BUILD_NUMBER + - value: 1 + - name: PYTORCH_REPO + - name: 'pytorch' + - name: PYTORCH_BRANCH + - name: 'v1.0.0' - ${{ if parameters.msagent }}: - name: USE_SCCACHE value: 0 diff --git a/windows/templates/linux_build_task.yml b/windows/templates/linux_build_task.yml index e8ecb5308..e0ef296f5 100644 --- a/windows/templates/linux_build_task.yml +++ b/windows/templates/linux_build_task.yml @@ -1,14 +1,12 @@ parameters: msagent: '' - cpuEnabled: '' - linuxEnabled: '' - condaEnabled: '' + enabled: False jobs: - job: 'Linux_CPU_Conda_Build' timeoutInMinutes: 0 cancelTimeoutInMinutes: 5 - condition: and({{ parameters.cpuEnabled }}, {{ parameters.linuxEnabled }}, {{ parameters.condaEnabled }}) + condition: {{ parameters.enabled }} variables: CUDA_VERSION: cpu TORCH_CONDA_BUILD_FOLDER: pytorch-nightly From 75db31903a973aa8b6a31e9b56e7bae92e771d80 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 17:34:17 +0800 Subject: [PATCH 24/30] Fix syntax error --- windows/templates/build_task.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml index accd53b8e..98b69f4c4 100644 --- a/windows/templates/build_task.yml +++ b/windows/templates/build_task.yml @@ -22,13 +22,13 @@ jobs: variables: - ${{ if parameters.override }}: - name: PYTORCH_BUILD_VERSION - - value: '1.0.0' + value: '1.0.0' - name: PYTORCH_BUILD_NUMBER - - value: 1 + value: 1 - name: PYTORCH_REPO - - name: 'pytorch' + value: 'pytorch' - name: PYTORCH_BRANCH - - name: 'v1.0.0' + value: 'v1.0.0' - ${{ if parameters.msagent }}: - name: USE_SCCACHE value: 0 From 72e467e19b679b871dbc5354f0dd941b39b09a40 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 17:35:37 +0800 Subject: [PATCH 25/30] . --- windows/templates/linux_build_task.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/windows/templates/linux_build_task.yml b/windows/templates/linux_build_task.yml index e0ef296f5..c1602475d 100644 --- a/windows/templates/linux_build_task.yml +++ b/windows/templates/linux_build_task.yml @@ -6,7 +6,7 @@ jobs: - job: 'Linux_CPU_Conda_Build' timeoutInMinutes: 0 cancelTimeoutInMinutes: 5 - condition: {{ parameters.enabled }} + condition: ${{ parameters.enabled }} variables: CUDA_VERSION: cpu TORCH_CONDA_BUILD_FOLDER: pytorch-nightly From 7a7044d8cabbd3ff1fedf1dc6fe335f2f93d07b0 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 18:03:50 +0800 Subject: [PATCH 26/30] Fix if conditions --- windows/templates/build_task.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml index 98b69f4c4..ad720fc6f 100644 --- a/windows/templates/build_task.yml +++ b/windows/templates/build_task.yml @@ -20,7 +20,7 @@ jobs: and(eq('${{ parameters.package }}', 'conda'), eq('${{ parameters.spec }}', 'cuda'), ${{ parameters.condaEnabled }}, ${{ parameters.cudaEnabled }}), and(eq('${{ parameters.package }}', 'wheels'), eq('${{ parameters.spec }}', 'cuda'), ${{ parameters.wheelsEnabled }}, ${{ parameters.cudaEnabled }})) variables: - - ${{ if parameters.override }}: + - ${{ if eq(parameters.override, True) }}: - name: PYTORCH_BUILD_VERSION value: '1.0.0' - name: PYTORCH_BUILD_NUMBER @@ -101,9 +101,9 @@ jobs: # CUDA_VERSION: 100 pool: - ${{ if parameters.msagent }}: + ${{ if eq(parameters.msagent, True) }}: vmImage: 'win1803' - ${{ if not(parameters.msagent) }}: + ${{ if eq(parameters.msagent, False) }}: name: 'pytorch' steps: From 607c612185ede6bf645ada7912b2280504715df8 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 20:31:31 +0800 Subject: [PATCH 27/30] Fail build if no CUDA and fix cpu upload --- windows/cuda100.bat | 8 +-- windows/cuda80.bat | 12 ++-- windows/cuda90.bat | 8 +-- windows/{ => old}/cuda91.bat | 94 ++++++++++++++++---------------- windows/{ => old}/cuda92.bat | 94 ++++++++++++++++---------------- windows/templates/build_task.yml | 8 ++- 6 files changed, 115 insertions(+), 109 deletions(-) rename windows/{ => old}/cuda91.bat (95%) mode change 100755 => 100644 rename windows/{ => old}/cuda92.bat (95%) mode change 100755 => 100644 diff --git a/windows/cuda100.bat b/windows/cuda100.bat index feb892acf..d4d1b7704 100644 --- a/windows/cuda100.bat +++ b/windows/cuda100.bat @@ -17,14 +17,14 @@ set NO_CUDA= set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 IF "%NVTOOLSEXT_PATH%"=="" ( - echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, disabling CUDA - set NO_CUDA=1 + echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing + exit /b 1 goto optcheck ) IF "%CUDA_PATH_V10_0%"=="" ( - echo CUDA 10.0 not found, disabling it - set NO_CUDA=1 + echo CUDA 10.0 not found, failing + exit /b 1 ) ELSE ( set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1;7.0;7.5 set TORCH_NVCC_FLAGS=-Xfatbin -compress-all diff --git a/windows/cuda80.bat b/windows/cuda80.bat index 25e0d3878..5e40cdb90 100755 --- a/windows/cuda80.bat +++ b/windows/cuda80.bat @@ -17,18 +17,18 @@ set NO_CUDA= set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 IF "%NVTOOLSEXT_PATH%"=="" ( - echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, disabling CUDA - set NO_CUDA=1 + echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing + exit /b 1 goto optcheck ) IF "%CUDA_PATH_V8_0%"=="" ( - echo CUDA 8 not found, disabling it - set NO_CUDA=1 + echo CUDA 8 not found, failing + exit /b 1 ) ELSE ( IF "%VS140COMNTOOLS%"=="" ( - echo CUDA 8 found, but VS2015 not found, disabling it - set NO_CUDA=1 + echo CUDA 8 found, but VS2015 not found, failing + exit /b 1 ) ELSE ( set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1 set TORCH_NVCC_FLAGS=-Xfatbin -compress-all diff --git a/windows/cuda90.bat b/windows/cuda90.bat index d9eedf4b7..42fdfd165 100755 --- a/windows/cuda90.bat +++ b/windows/cuda90.bat @@ -17,14 +17,14 @@ set NO_CUDA= set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 IF "%NVTOOLSEXT_PATH%"=="" ( - echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, disabling CUDA - set NO_CUDA=1 + echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, failing + exit /b 1 goto optcheck ) IF "%CUDA_PATH_V9_0%"=="" ( - echo CUDA 9 not found, disabling it - set NO_CUDA=1 + echo CUDA 9 not found, failing + exit /b 1 ) ELSE ( set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;7.0 set TORCH_NVCC_FLAGS=-Xfatbin -compress-all diff --git a/windows/cuda91.bat b/windows/old/cuda91.bat old mode 100755 new mode 100644 similarity index 95% rename from windows/cuda91.bat rename to windows/old/cuda91.bat index fd31716df..c1a3f8ca9 --- a/windows/cuda91.bat +++ b/windows/old/cuda91.bat @@ -1,47 +1,47 @@ -@echo off - -IF NOT EXIST "setup.py" IF NOT EXIST "pytorch" ( - call internal\clone.bat - cd .. - IF ERRORLEVEL 1 goto eof -) ELSE ( - call internal\clean.bat -) - -call internal\check_deps.bat -IF ERRORLEVEL 1 goto eof - -REM Check for optional components - -set NO_CUDA= -set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 - -IF "%NVTOOLSEXT_PATH%"=="" ( - echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, disabling CUDA - set NO_CUDA=1 - goto optcheck -) - -IF "%CUDA_PATH_V9_1%"=="" ( - echo CUDA 9.1 not found, disabling it - set NO_CUDA=1 -) ELSE ( - set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1;7.0 - set TORCH_NVCC_FLAGS=-Xfatbin -compress-all - - set "CUDA_PATH=%CUDA_PATH_V9_1%" - set "PATH=%CUDA_PATH_V9_1%\bin;%PATH%" -) - -:optcheck - -call internal\check_opts.bat -IF ERRORLEVEL 1 goto eof - -call internal\copy.bat -IF ERRORLEVEL 1 goto eof - -call internal\setup.bat -IF ERRORLEVEL 1 goto eof - -:eof +@echo off + +IF NOT EXIST "setup.py" IF NOT EXIST "pytorch" ( + call internal\clone.bat + cd .. + IF ERRORLEVEL 1 goto eof +) ELSE ( + call internal\clean.bat +) + +call internal\check_deps.bat +IF ERRORLEVEL 1 goto eof + +REM Check for optional components + +set NO_CUDA= +set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 + +IF "%NVTOOLSEXT_PATH%"=="" ( + echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, disabling CUDA + set NO_CUDA=1 + goto optcheck +) + +IF "%CUDA_PATH_V9_1%"=="" ( + echo CUDA 9.1 not found, disabling it + set NO_CUDA=1 +) ELSE ( + set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1;7.0 + set TORCH_NVCC_FLAGS=-Xfatbin -compress-all + + set "CUDA_PATH=%CUDA_PATH_V9_1%" + set "PATH=%CUDA_PATH_V9_1%\bin;%PATH%" +) + +:optcheck + +call internal\check_opts.bat +IF ERRORLEVEL 1 goto eof + +call internal\copy.bat +IF ERRORLEVEL 1 goto eof + +call internal\setup.bat +IF ERRORLEVEL 1 goto eof + +:eof diff --git a/windows/cuda92.bat b/windows/old/cuda92.bat old mode 100755 new mode 100644 similarity index 95% rename from windows/cuda92.bat rename to windows/old/cuda92.bat index 0d97f4629..ca8cd4430 --- a/windows/cuda92.bat +++ b/windows/old/cuda92.bat @@ -1,47 +1,47 @@ -@echo off - -IF NOT EXIST "setup.py" IF NOT EXIST "pytorch" ( - call internal\clone.bat - cd .. - IF ERRORLEVEL 1 goto eof -) ELSE ( - call internal\clean.bat -) - -call internal\check_deps.bat -IF ERRORLEVEL 1 goto eof - -REM Check for optional components - -set NO_CUDA= -set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 - -IF "%NVTOOLSEXT_PATH%"=="" ( - echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, disabling CUDA - set NO_CUDA=1 - goto optcheck -) - -IF "%CUDA_PATH_V9_2%"=="" ( - echo CUDA 9.2 not found, disabling it - set NO_CUDA=1 -) ELSE ( - set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1;7.0 - set TORCH_NVCC_FLAGS=-Xfatbin -compress-all - - set "CUDA_PATH=%CUDA_PATH_V9_2%" - set "PATH=%CUDA_PATH_V9_2%\bin;%PATH%" -) - -:optcheck - -call internal\check_opts.bat -IF ERRORLEVEL 1 goto eof - -call internal\copy.bat -IF ERRORLEVEL 1 goto eof - -call internal\setup.bat -IF ERRORLEVEL 1 goto eof - -:eof +@echo off + +IF NOT EXIST "setup.py" IF NOT EXIST "pytorch" ( + call internal\clone.bat + cd .. + IF ERRORLEVEL 1 goto eof +) ELSE ( + call internal\clean.bat +) + +call internal\check_deps.bat +IF ERRORLEVEL 1 goto eof + +REM Check for optional components + +set NO_CUDA= +set CMAKE_GENERATOR=Visual Studio 15 2017 Win64 + +IF "%NVTOOLSEXT_PATH%"=="" ( + echo NVTX ^(Visual Studio Extension ^for CUDA^) ^not installed, disabling CUDA + set NO_CUDA=1 + goto optcheck +) + +IF "%CUDA_PATH_V9_2%"=="" ( + echo CUDA 9.2 not found, disabling it + set NO_CUDA=1 +) ELSE ( + set TORCH_CUDA_ARCH_LIST=3.5;5.0+PTX;6.0;6.1;7.0 + set TORCH_NVCC_FLAGS=-Xfatbin -compress-all + + set "CUDA_PATH=%CUDA_PATH_V9_2%" + set "PATH=%CUDA_PATH_V9_2%\bin;%PATH%" +) + +:optcheck + +call internal\check_opts.bat +IF ERRORLEVEL 1 goto eof + +call internal\copy.bat +IF ERRORLEVEL 1 goto eof + +call internal\setup.bat +IF ERRORLEVEL 1 goto eof + +:eof diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml index ad720fc6f..6daa6ecfa 100644 --- a/windows/templates/build_task.yml +++ b/windows/templates/build_task.yml @@ -131,7 +131,13 @@ jobs: parameters: msagent: ${{ parameters.msagent }} - - ${{ if eq(parameters.package, 'wheels') }}: + - ${{ if and(eq(parameters.package, 'wheels'), eq(parameters.spec, 'cpu')) }}: + - template: upload_to_s3.yml + parameters: + cuVer: '$(CUDA_VERSION)' + cudaVer: '$(CUDA_VERSION)' + + - ${{ if and(eq(parameters.package, 'wheels'), ne(parameters.spec, 'cpu')) }}: - template: upload_to_s3.yml parameters: cuVer: 'cu$(CUDA_VERSION)' From 29b81c7d2eed4dc5ab681cb753c85820c4232e52 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 20:55:25 +0800 Subject: [PATCH 28/30] Cleanup --- windows/azure-pipelines-legacy.yml | 309 ----------------------------- windows/azure-pipelines.yml | 24 +-- windows/templates/build_task.yml | 112 +++++------ 3 files changed, 64 insertions(+), 381 deletions(-) delete mode 100644 windows/azure-pipelines-legacy.yml diff --git a/windows/azure-pipelines-legacy.yml b/windows/azure-pipelines-legacy.yml deleted file mode 100644 index e886737e4..000000000 --- a/windows/azure-pipelines-legacy.yml +++ /dev/null @@ -1,309 +0,0 @@ - -# Turn off auto builds for commits -trigger: none -pr: none - -variables: - # pytorch.build.version: '1.0.0' - # pytorch.build.number: 1 - # pytorch.branch: master - # pytorch.repo: pytorch - build.linux.enabled: false - build.cpu.enabled: true - build.cuda.enabled: true - build.wheels.enabled: true - build.conda.enabled: true - use.sccache.cpu: 0 - use.sccache.cuda: 1 - self.hosted.agent.cpu: false - self.hosted.agent.cuda: true - -jobs: -- job: 'VSTS_Auth_Task' - timeoutInMinutes: 5 - cancelTimeoutInMinutes: 5 - variables: - - group: 'peterjc-vsts-token' - - pool: - vmImage: 'win1803' - - steps: - - checkout: self - clean: true - - - template: templates/vsts_auth.yml - parameters: - auth: $(vsts_auth) - -- job: 'Windows_CPU_Wheels_Build' - timeoutInMinutes: 0 - cancelTimeoutInMinutes: 5 - condition: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) - variables: - CUDA_VERSION: cpu - USE_SCCACHE: $(use.sccache.cpu) - strategy: - maxParallel: 10 - matrix: - PY3.5: - DESIRED_PYTHON: 3.5 - PY3.6: - DESIRED_PYTHON: 3.6 - PY3.7: - DESIRED_PYTHON: 3.7 - LIBTORCH: - DESIRED_PYTHON: 3 - BUILD_PYTHONLESS: 1 - - pool: - vmImage: 'win1803' - - steps: - - checkout: self - clean: true - - - template: templates/setup_env_for_msagent.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/setup_nightly_variables.yml - - - template: templates/build_wheels.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/publish_test_results.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/upload_to_s3.yml - parameters: - cuVer: 'cpu' - cudaVer: 'cpu' - -- job: 'Windows_CPU_Conda_Build' - timeoutInMinutes: 0 - cancelTimeoutInMinutes: 5 - condition: and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) - variables: - - group: peterjc_anaconda_token - - name: CUDA_VERSION - value: cpu - - name: USE_SCCACHE - value: $(use.sccache.cpu) - - name: PYTORCH_FINAL_PACKAGE_DIR - value: '$(Build.Repository.LocalPath)\windows\output' - - strategy: - maxParallel: 10 - matrix: - PY3.5: - DESIRED_PYTHON: 3.5 - PY3.6: - DESIRED_PYTHON: 3.6 - PY3.7: - DESIRED_PYTHON: 3.7 - - pool: - vmImage: 'win1803' - - steps: - - checkout: self - clean: true - - - template: templates/setup_env_for_msagent.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/setup_nightly_variables.yml - - - template: templates/build_conda.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/publish_test_results.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/upload_to_conda.yml - parameters: - user: $(peterjc_conda_username) - pass: $(peterjc_conda_password) - -- job: 'Linux_CPU_Conda_Build' - timeoutInMinutes: 0 - cancelTimeoutInMinutes: 5 - condition: and(and(eq(variables['build.cpu.enabled'], 'true'), eq(variables['build.linux.enabled'], 'true')), eq(variables['build.conda.enabled'], 'true')) - variables: - CUDA_VERSION: cpu - TORCH_CONDA_BUILD_FOLDER: pytorch-nightly - PYTORCH_FINAL_PACKAGE_DIR: '$(Build.Repository.LocalPath)/output' - - strategy: - maxParallel: 10 - matrix: - PY3.5: - DESIRED_PYTHON: 3.5 - - pool: - vmImage: 'ubuntu-16.04' - - steps: - - checkout: self - clean: true - - - script: 'sudo apt-get install p7zip-full' - displayName: 'Install 7Zip' - - - task: CondaEnvironment@1 - displayName: 'Install conda-build' - inputs: - packageSpecs: 'conda-build' - - - template: templates/build_conda.yml - parameters: - msagent: $(self.hosted.agent.cpu) - -- job: 'Windows_CUDA_Wheels_Build' - timeoutInMinutes: 0 - cancelTimeoutInMinutes: 5 - condition: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.wheels.enabled'], 'true')) - variables: - USE_SCCACHE: $(use.sccache.cuda) - strategy: - maxParallel: 10 - matrix: - PY3.5_90: - DESIRED_PYTHON: 3.5 - CUDA_VERSION: 90 - PY3.6_90: - DESIRED_PYTHON: 3.6 - CUDA_VERSION: 90 - PY3.7_90: - DESIRED_PYTHON: 3.7 - CUDA_VERSION: 90 - LIBTORCH_90: - DESIRED_PYTHON: 3 - BUILD_PYTHONLESS: 1 - CUDA_VERSION: 90 - PY3.5_80: - DESIRED_PYTHON: 3.5 - CUDA_VERSION: 80 - PY3.6_80: - DESIRED_PYTHON: 3.6 - CUDA_VERSION: 80 - PY3.7_80: - DESIRED_PYTHON: 3.7 - CUDA_VERSION: 80 - LIBTORCH_80: - DESIRED_PYTHON: 3 - BUILD_PYTHONLESS: 1 - CUDA_VERSION: 80 - PY3.5_100: - DESIRED_PYTHON: 3.5 - CUDA_VERSION: 100 - PY3.6_100: - DESIRED_PYTHON: 3.6 - CUDA_VERSION: 100 - PY3.7_100: - DESIRED_PYTHON: 3.7 - CUDA_VERSION: 100 - LIBTORCH_100: - DESIRED_PYTHON: 3 - BUILD_PYTHONLESS: 1 - CUDA_VERSION: 100 - - pool: - name: 'pytorch' - - steps: - - checkout: self - clean: true - - - template: templates/setup_env_for_msagent.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/setup_nightly_variables.yml - - - template: templates/build_wheels.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/publish_test_results.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/upload_to_s3.yml - parameters: - cuVer: 'cu$(CUDA_VERSION)' - cudaVer: 'cuda$(CUDA_VERSION)' - -- job: 'Windows_CUDA_Conda_Build' - timeoutInMinutes: 0 - condition: and(eq(variables['build.cuda.enabled'], 'true'), eq(variables['build.conda.enabled'], 'true')) - cancelTimeoutInMinutes: 5 - variables: - - group: peterjc_anaconda_token - - name: USE_SCCACHE - value: $(use.sccache.cuda) - - name: PYTORCH_FINAL_PACKAGE_DIR - value: '$(Build.Repository.LocalPath)\windows\output' - - strategy: - maxParallel: 10 - matrix: - PY3.5_90: - DESIRED_PYTHON: 3.5 - CUDA_VERSION: 90 - PY3.6_90: - DESIRED_PYTHON: 3.6 - CUDA_VERSION: 90 - PY3.7_90: - DESIRED_PYTHON: 3.7 - CUDA_VERSION: 90 - PY3.5_80: - DESIRED_PYTHON: 3.5 - CUDA_VERSION: 80 - PY3.6_80: - DESIRED_PYTHON: 3.6 - CUDA_VERSION: 80 - PY3.7_80: - DESIRED_PYTHON: 3.7 - CUDA_VERSION: 80 - PY3.5_100: - DESIRED_PYTHON: 3.5 - CUDA_VERSION: 100 - PY3.6_100: - DESIRED_PYTHON: 3.6 - CUDA_VERSION: 100 - PY3.7_100: - DESIRED_PYTHON: 3.7 - CUDA_VERSION: 100 - - pool: - name: 'pytorch' - - steps: - - checkout: self - clean: true - - - template: templates/setup_env_for_msagent.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/setup_nightly_variables.yml - - - template: templates/build_conda.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/publish_test_results.yml - parameters: - msagent: $(self.hosted.agent.cpu) - - - template: templates/upload_to_conda.yml - parameters: - user: $(peterjc_conda_username) - pass: $(peterjc_conda_password) diff --git a/windows/azure-pipelines.yml b/windows/azure-pipelines.yml index 97fa60346..07ab8928a 100644 --- a/windows/azure-pipelines.yml +++ b/windows/azure-pipelines.yml @@ -8,34 +8,26 @@ jobs: - template: templates/build_task.yml parameters: - package: 'wheels' - spec: 'cpu' - jobDesc: 'Wheels' - packageDesc: 'CPU' + package: 'Wheels' + spec: 'CPU' msagent: True - template: templates/build_task.yml parameters: - package: 'conda' - spec: 'cpu' - jobDesc: 'Conda' - packageDesc: 'CPU' + package: 'Conda' + spec: 'CPU' msagent: True - template: templates/build_task.yml parameters: - package: 'wheels' - spec: 'cuda' - jobDesc: 'Wheels' - packageDesc: 'CUDA' + package: 'Wheels' + spec: 'CUDA' msagent: False - template: templates/build_task.yml parameters: - package: 'conda' - spec: 'cuda' - jobDesc: 'Conda' - packageDesc: 'CUDA' + package: 'Conda' + spec: 'CUDA' msagent: False - template: templates/linux_build_task.yml diff --git a/windows/templates/build_task.yml b/windows/templates/build_task.yml index 6daa6ecfa..f61667429 100644 --- a/windows/templates/build_task.yml +++ b/windows/templates/build_task.yml @@ -11,14 +11,14 @@ parameters: override: False jobs: -- job: 'Windows_${{ parameters.jobDesc }}_${{ parameters.packageDesc }}_Build' +- job: 'Windows_${{ parameters.spec }}_${{ parameters.package }}_Build' timeoutInMinutes: 0 cancelTimeoutInMinutes: 5 condition: > - or(and(eq('${{ parameters.package }}', 'conda'), eq('${{ parameters.spec }}', 'cpu'), ${{ parameters.condaEnabled }}, ${{ parameters.cpuEnabled }}), - and(eq('${{ parameters.package }}', 'wheels'), eq('${{ parameters.spec }}', 'cpu'), ${{ parameters.wheelsEnabled }}, ${{ parameters.cpuEnabled }}), - and(eq('${{ parameters.package }}', 'conda'), eq('${{ parameters.spec }}', 'cuda'), ${{ parameters.condaEnabled }}, ${{ parameters.cudaEnabled }}), - and(eq('${{ parameters.package }}', 'wheels'), eq('${{ parameters.spec }}', 'cuda'), ${{ parameters.wheelsEnabled }}, ${{ parameters.cudaEnabled }})) + or(and(eq('${{ parameters.package }}', 'Conda'), eq('${{ parameters.spec }}', 'CPU'), ${{ parameters.condaEnabled }}, ${{ parameters.cpuEnabled }}), + and(eq('${{ parameters.package }}', 'Wheels'), eq('${{ parameters.spec }}', 'CPU'), ${{ parameters.wheelsEnabled }}, ${{ parameters.cpuEnabled }}), + and(eq('${{ parameters.package }}', 'Conda'), eq('${{ parameters.spec }}', 'CUDA'), ${{ parameters.condaEnabled }}, ${{ parameters.cudaEnabled }}), + and(eq('${{ parameters.package }}', 'Wheels'), eq('${{ parameters.spec }}', 'CUDA'), ${{ parameters.wheelsEnabled }}, ${{ parameters.cudaEnabled }})) variables: - ${{ if eq(parameters.override, True) }}: - name: PYTORCH_BUILD_VERSION @@ -29,13 +29,13 @@ jobs: value: 'pytorch' - name: PYTORCH_BRANCH value: 'v1.0.0' - - ${{ if parameters.msagent }}: + - ${{ if eq(parameters.msagent, True) }}: - name: USE_SCCACHE value: 0 - - ${{ if not(parameters.msagent) }}: + - ${{ if eq(parameters.msagent, False) }}: - name: USE_SCCACHE value: 1 - - ${{ if eq(parameters.package, 'conda') }}: + - ${{ if eq(parameters.package, 'Conda') }}: - group: peterjc_anaconda_token - name: PYTORCH_FINAL_PACKAGE_DIR value: '$(Build.Repository.LocalPath)\windows\output' @@ -43,62 +43,62 @@ jobs: strategy: maxParallel: 10 matrix: - ${{ if eq(parameters.spec, 'cpu') }}: + ${{ if eq(parameters.spec, 'CPU') }}: PY3.5: DESIRED_PYTHON: 3.5 CUDA_VERSION: cpu - # PY3.6: - # DESIRED_PYTHON: 3.6 - # CUDA_VERSION: cpu - # PY3.7: - # DESIRED_PYTHON: 3.7 - # CUDA_VERSION: cpu - ${{ if and(eq(parameters.package, 'wheels'), eq(parameters.spec, 'cpu')) }}: + PY3.6: + DESIRED_PYTHON: 3.6 + CUDA_VERSION: cpu + PY3.7: + DESIRED_PYTHON: 3.7 + CUDA_VERSION: cpu + ${{ if and(eq(parameters.package, 'Wheels'), eq(parameters.spec, 'CPU')) }}: LIBTORCH: DESIRED_PYTHON: 3 CUDA_VERSION: cpu BUILD_PYTHONLESS: 1 - ${{ if ne(parameters.spec, 'cpu') }}: + ${{ if ne(parameters.spec, 'CPU') }}: PY3.5_90: DESIRED_PYTHON: 3.5 CUDA_VERSION: 90 - # PY3.6_90: - # DESIRED_PYTHON: 3.6 - # CUDA_VERSION: 90 - # PY3.7_90: - # DESIRED_PYTHON: 3.7 - # CUDA_VERSION: 90 - # PY3.5_80: - # DESIRED_PYTHON: 3.5 - # CUDA_VERSION: 80 - # PY3.6_80: - # DESIRED_PYTHON: 3.6 - # CUDA_VERSION: 80 - # PY3.7_80: - # DESIRED_PYTHON: 3.7 - # CUDA_VERSION: 80 - # PY3.5_100: - # DESIRED_PYTHON: 3.5 - # CUDA_VERSION: 100 - # PY3.6_100: - # DESIRED_PYTHON: 3.6 - # CUDA_VERSION: 100 - # PY3.7_100: - # DESIRED_PYTHON: 3.7 - # CUDA_VERSION: 100 - ${{ if and(ne(parameters.spec, 'cpu'), eq(parameters.package, 'wheels')) }}: - # LIBTORCH_80: - # DESIRED_PYTHON: 3 - # BUILD_PYTHONLESS: 1 - # CUDA_VERSION: 80 + PY3.6_90: + DESIRED_PYTHON: 3.6 + CUDA_VERSION: 90 + PY3.7_90: + DESIRED_PYTHON: 3.7 + CUDA_VERSION: 90 + PY3.5_80: + DESIRED_PYTHON: 3.5 + CUDA_VERSION: 80 + PY3.6_80: + DESIRED_PYTHON: 3.6 + CUDA_VERSION: 80 + PY3.7_80: + DESIRED_PYTHON: 3.7 + CUDA_VERSION: 80 + PY3.5_100: + DESIRED_PYTHON: 3.5 + CUDA_VERSION: 100 + PY3.6_100: + DESIRED_PYTHON: 3.6 + CUDA_VERSION: 100 + PY3.7_100: + DESIRED_PYTHON: 3.7 + CUDA_VERSION: 100 + ${{ if and(ne(parameters.spec, 'CPU'), eq(parameters.package, 'Wheels')) }}: + LIBTORCH_80: + DESIRED_PYTHON: 3 + BUILD_PYTHONLESS: 1 + CUDA_VERSION: 80 LIBTORCH_90: DESIRED_PYTHON: 3 BUILD_PYTHONLESS: 1 CUDA_VERSION: 90 - # LIBTORCH_100: - # DESIRED_PYTHON: 3 - # BUILD_PYTHONLESS: 1 - # CUDA_VERSION: 100 + LIBTORCH_100: + DESIRED_PYTHON: 3 + BUILD_PYTHONLESS: 1 + CUDA_VERSION: 100 pool: ${{ if eq(parameters.msagent, True) }}: @@ -116,34 +116,34 @@ jobs: - template: setup_nightly_variables.yml - - ${{ if eq(parameters.package, 'wheels') }}: + - ${{ if eq(parameters.package, 'Wheels') }}: - template: build_wheels.yml parameters: msagent: ${{ parameters.msagent }} - - ${{ if eq(parameters.package, 'conda') }}: + - ${{ if eq(parameters.package, 'Conda') }}: - template: build_conda.yml parameters: msagent: ${{ parameters.msagent }} - - ${{ if or(eq(parameters.package, 'wheels'), eq(parameters.package, 'conda')) }}: + - ${{ if or(eq(parameters.package, 'Wheels'), eq(parameters.package, 'Conda')) }}: - template: publish_test_results.yml parameters: msagent: ${{ parameters.msagent }} - - ${{ if and(eq(parameters.package, 'wheels'), eq(parameters.spec, 'cpu')) }}: + - ${{ if and(eq(parameters.package, 'Wheels'), eq(parameters.spec, 'CPU')) }}: - template: upload_to_s3.yml parameters: cuVer: '$(CUDA_VERSION)' cudaVer: '$(CUDA_VERSION)' - - ${{ if and(eq(parameters.package, 'wheels'), ne(parameters.spec, 'cpu')) }}: + - ${{ if and(eq(parameters.package, 'Wheels'), ne(parameters.spec, 'CPU')) }}: - template: upload_to_s3.yml parameters: cuVer: 'cu$(CUDA_VERSION)' cudaVer: 'cuda$(CUDA_VERSION)' - - ${{ if eq(parameters.package, 'conda') }}: + - ${{ if eq(parameters.package, 'Conda') }}: - template: upload_to_conda.yml parameters: user: $(peterjc_conda_username) From 97817c913e9231db7b2b4b696fd11804ae093746 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 20:58:13 +0800 Subject: [PATCH 29/30] Unify style --- windows/templates/vsts_auth.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/windows/templates/vsts_auth.yml b/windows/templates/vsts_auth.yml index e83a444f3..5a036ce63 100644 --- a/windows/templates/vsts_auth.yml +++ b/windows/templates/vsts_auth.yml @@ -2,7 +2,7 @@ parameters: auth: '' steps: - - script: 'call windows/internal/auth.bat' - displayName: 'Sign in to Azure Pipelines' - env: - VSTS_AUTH: ${{ parameters.auth }} +- script: 'call windows/internal/auth.bat' + displayName: 'Sign in to Azure Pipelines' + env: + VSTS_AUTH: ${{ parameters.auth }} From 3a91e1fa1f4683898d57ffd05b9475562e52f00a Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 25 Dec 2018 21:02:27 +0800 Subject: [PATCH 30/30] Unify if style --- windows/templates/build_conda.yml | 2 +- windows/templates/build_wheels.yml | 2 +- windows/templates/setup_env_for_msagent.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/windows/templates/build_conda.yml b/windows/templates/build_conda.yml index 7b228c8e0..383c4ef3d 100644 --- a/windows/templates/build_conda.yml +++ b/windows/templates/build_conda.yml @@ -8,5 +8,5 @@ steps: - bash: './conda/build_pytorch.sh $CUDA_VERSION $PYTORCH_BUILD_VERSION $PYTORCH_BUILD_NUMBER' displayName: Build env: - ${{ if parameters.msagent }}: + ${{ if eq(parameters.msagent, True) }}: MAX_JOBS: 2 diff --git a/windows/templates/build_wheels.yml b/windows/templates/build_wheels.yml index 151a7c871..1e9ee74fe 100644 --- a/windows/templates/build_wheels.yml +++ b/windows/templates/build_wheels.yml @@ -5,5 +5,5 @@ steps: - script: 'call windows/build_pytorch.bat %CUDA_VERSION% %PYTORCH_BUILD_VERSION% %PYTORCH_BUILD_NUMBER%' displayName: Build env: - ${{ if parameters.msagent }}: + ${{ if eq(parameters.msagent, True) }}: MAX_JOBS: 2 diff --git a/windows/templates/setup_env_for_msagent.yml b/windows/templates/setup_env_for_msagent.yml index b87e2da03..832842b2a 100644 --- a/windows/templates/setup_env_for_msagent.yml +++ b/windows/templates/setup_env_for_msagent.yml @@ -2,7 +2,7 @@ parameters: msagent: false steps: -- ${{ if parameters.msagent }}: +- ${{ if eq(parameters.msagent, True) }}: - task: BatchScript@1 displayName: 'Install VS 2017' inputs: