Update base for Update on "assert no duplicate yaml keys in codegen"

The codegen should error if it sees two yaml entries with the same key. The default behavior of python's yaml loader is to overwrite duplicate keys with the new value (see yaml/pyyaml#165). This would have caught a nasty bug that showed up in https://github.com/pytorch/pytorch/pull/66225/files#r723796194. I tested it on that linked PR, to confirm that it errors correctly (and gives the line number containing the duplicate). Differential Revision: [D31464585](https://our.internmc.facebook.com/intern/diff/D31464585) [ghstack-poisoned]
pytorch · Oct 13, 2021 · eb48590 · eb48590
2 parents df475aa + ecb7b38
commit eb48590
Show file tree

Hide file tree

Showing 759 changed files with 22,955 additions and 7,981 deletions.
diff --git a/.azure_pipelines/job_templates/prepare-build-template.yml b/.azure_pipelines/job_templates/prepare-build-template.yml
@@ -46,7 +46,7 @@ steps:
       curl -k https://s3.amazonaws.com/ossci-windows/sccache.exe --output .\tmp_bin\sccache.exe
       curl -k https://s3.amazonaws.com/ossci-windows/sccache-cl.exe --output .\tmp_bin\sccache-cl.exe
       copy .\tmp_bin\sccache.exe .\tmp_bin\nvcc.exe
-      curl -kL https://github.com/peterjc123/randomtemp-rust/releases/download/v0.3/randomtemp.exe --output .\tmp_bin\randomtemp.exe
+      curl -kL https://github.com/peterjc123/randomtemp-rust/releases/download/v0.4/randomtemp.exe --output .\tmp_bin\randomtemp.exe
     displayName: Install sccache and randomtemp
     condition: not(eq(variables.CUDA_VERSION, ''))
 

diff --git a/.azure_pipelines/job_templates/set-environment-variables.yml b/.azure_pipelines/job_templates/set-environment-variables.yml
@@ -120,9 +120,7 @@ steps:
         Write-Host "##vso[task.setvariable variable=CMAKE_LIBRARY_PATH;]$(Build.SourcesDirectory)\mkl\lib;$env:CMAKE_LIBRARY_PATH"
         Write-Host "##vso[task.setvariable variable=ADDITIONAL_PATH;]$(Build.SourcesDirectory)\tmp_bin"
         Write-Host "##vso[task.setvariable variable=SCCACHE_IDLE_TIMEOUT;]1500"
-        Write-Host "##vso[task.setvariable variable=RANDOMTEMP_EXECUTABLE;]$(Build.SourcesDirectory)\tmp_bin\nvcc.exe"
-        Write-Host "##vso[task.setvariable variable=CUDA_NVCC_EXECUTABLE;]$(Build.SourcesDirectory)\tmp_bin\randomtemp.exe"
-        Write-Host "##vso[task.setvariable variable=RANDOMTEMP_BASEDIR;]$(Build.SourcesDirectory)\tmp_bin"
+        Write-Host "##vso[task.setvariable variable=CMAKE_CUDA_COMPILER_LAUNCHER;]$(Build.SourcesDirectory)/tmp_bin/randomtemp.exe;$(Build.SourcesDirectory)/tmp_bin/sccache.exe"
       displayName: Set MKL, sccache and randomtemp environment variables
 
     # View current environment variables

diff --git a/.circleci/cimodel/data/pytorch_build_data.py b/.circleci/cimodel/data/pytorch_build_data.py
@@ -12,13 +12,6 @@
             # TODO: bring back libtorch test
             ("7", [X("3.6")]),
         ]),
-        ("clang", [
-            ("7", [
-                ("3.6", [
-                    ("onnx", [XImportant(True)]),
-                ]),
-            ]),
-        ]),
         ("cuda", [
             ("10.2", [
                 ("3.6", [
@@ -47,7 +40,6 @@
             ("9", [
                 ("3.6", [
                     ("xla", [XImportant(True)]),
-                    ("vulkan", [XImportant(True)]),
                 ]),
             ]),
         ]),

diff --git a/.circleci/cimodel/data/pytorch_build_definitions.py b/.circleci/cimodel/data/pytorch_build_definitions.py
@@ -352,28 +352,6 @@ def instantiate_configs(only_slow_gradcheck):
                                         tags_list=RC_PATTERN)
             c.dependent_tests = gen_docs_configs(c)
 
-        if (
-            compiler_name != "clang"
-            and not rocm_version
-            and not is_libtorch
-            and not is_vulkan
-            and not is_pure_torch
-            and not is_noarch
-            and not is_slow_gradcheck
-            and not only_slow_gradcheck
-            and not build_only
-        ):
-            distributed_test = Conf(
-                c.gen_build_name("") + "distributed",
-                [],
-                is_xla=False,
-                restrict_phases=["test"],
-                is_libtorch=False,
-                is_important=True,
-                parent_build=c,
-            )
-            c.dependent_tests.append(distributed_test)
-
         config_list.append(c)
 
     return config_list

diff --git a/.circleci/config.yml b/.circleci/config.yml
diff --git a/.circleci/docker/build_docker.sh b/.circleci/docker/build_docker.sh
@@ -26,11 +26,14 @@ login() {
     docker login -u AWS --password-stdin "$1"
 }
 
-# Retry on timeouts (can happen on job stampede).
-retry login "${registry}"
 
-# Logout on exit
-trap "docker logout ${registry}" EXIT
+# Only run these steps if not on github actions
+if [[ -z "${GITHUB_ACTIONS}" ]]; then
+  # Retry on timeouts (can happen on job stampede).
+  retry login "${registry}"
+  # Logout on exit
+  trap "docker logout ${registry}" EXIT
+fi
 
 # export EC2=1
 # export JENKINS=1
@@ -45,8 +48,8 @@ trap "docker logout ${registry}" EXIT
 
 docker push "${image}:${tag}"
 
-docker save -o "${IMAGE_NAME}:${tag}.tar" "${image}:${tag}"
-
 if [ -z "${DOCKER_SKIP_S3_UPLOAD:-}" ]; then
+  trap "rm -rf ${IMAGE_NAME}:${tag}.tar" EXIT
+  docker save -o "${IMAGE_NAME}:${tag}.tar" "${image}:${tag}"
   aws s3 cp "${IMAGE_NAME}:${tag}.tar" "s3://ossci-linux-build/pytorch/base/${IMAGE_NAME}:${tag}.tar" --acl public-read
 fi
diff --git a/.circleci/docker/common/install_base.sh b/.circleci/docker/common/install_base.sh
@@ -44,6 +44,10 @@ install_ubuntu() {
     wget \
     vim
 
+  # Should resolve issues related to various apt package repository cert issues
+  # see: https://github.com/pytorch/pytorch/issues/65931
+  apt-get install -y libgnutls30
+
   # Cleanup package manager
   apt-get autoclean && apt-get clean
   rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
@@ -109,10 +113,7 @@ esac
 # Install Valgrind separately since the apt-get version is too old.
 mkdir valgrind_build && cd valgrind_build
 VALGRIND_VERSION=3.16.1
-if ! wget http://valgrind.org/downloads/valgrind-${VALGRIND_VERSION}.tar.bz2
-then
-  wget https://sourceware.org/ftp/valgrind/valgrind-${VALGRIND_VERSION}.tar.bz2
-fi
+wget https://ossci-linux.s3.amazonaws.com/valgrind-${VALGRIND_VERSION}.tar.bz2
 tar -xjf valgrind-${VALGRIND_VERSION}.tar.bz2
 cd valgrind-${VALGRIND_VERSION}
 ./configure --prefix=/usr/local

diff --git a/.circleci/docker/common/install_openssl.sh b/.circleci/docker/common/install_openssl.sh
@@ -4,7 +4,7 @@ set -ex
 
 OPENSSL=openssl-1.1.1k
 
-wget -q -O "${OPENSSL}.tar.gz" "https://www.openssl.org/source/${OPENSSL}.tar.gz"
+wget -q -O "${OPENSSL}.tar.gz" "https://ossci-linux.s3.amazonaws.com/${OPENSSL}.tar.gz"
 tar xf "${OPENSSL}.tar.gz"
 cd "${OPENSSL}"
 ./config --prefix=/opt/openssl -d '-Wl,--enable-new-dtags,-rpath,$(LIBRPATH)'

diff --git a/.circleci/docker/ubuntu-cuda/Dockerfile b/.circleci/docker/ubuntu-cuda/Dockerfile
@@ -75,7 +75,7 @@ RUN rm install_cmake.sh
 ADD ./common/install_cache.sh install_cache.sh
 ENV PATH /opt/cache/bin:$PATH
 RUN bash ./install_cache.sh && rm install_cache.sh
-ENV CUDA_NVCC_EXECUTABLE=/opt/cache/lib/nvcc
+ENV CMAKE_CUDA_COMPILER_LAUNCHER=/opt/cache/bin/sccache
 
 # Add jni.h for java host build
 ADD ./common/install_jni.sh install_jni.sh
@@ -94,6 +94,7 @@ ENV BUILD_ENVIRONMENT ${BUILD_ENVIRONMENT}
 # AWS specific CUDA build guidance
 ENV TORCH_CUDA_ARCH_LIST Maxwell
 ENV TORCH_NVCC_FLAGS "-Xfatbin -compress-all"
+ENV CUDA_PATH /usr/local/cuda
 
 # Install LLVM dev version (Defined in the pytorch/builder github repository)
 COPY --from=pytorch/llvm:9.0.1 /opt/llvm /opt/llvm

diff --git a/.circleci/generate_config_yml.py b/.circleci/generate_config_yml.py
@@ -169,17 +169,6 @@ def gen_build_workflows_tree():
         binary_build_definitions.get_nightly_uploads,
     ]
 
-    slow_gradcheck_functions = [
-        pytorch_build_definitions.get_workflow_jobs
-    ]
-    slow_gradcheck_jobs = [f(only_slow_gradcheck=True) for f in slow_gradcheck_functions]
-    slow_gradcheck_jobs.extend(
-        cimodel.data.simple.docker_definitions.get_workflow_jobs(
-            # sort for consistency
-            sorted(generate_required_docker_images(slow_gradcheck_jobs))
-        )
-    )
-
     return {
         "workflows": {
             "binary_builds": {
@@ -194,10 +183,6 @@ def gen_build_workflows_tree():
                 "when": r"<< pipeline.parameters.run_master_build >>",
                 "jobs": master_build_jobs,
             },
-            "slow_gradcheck_build": {
-                "when": r"<< pipeline.parameters.run_slow_gradcheck_build >>",
-                "jobs": slow_gradcheck_jobs,
-            },
         }
     }
 
@@ -221,7 +206,6 @@ def gen_build_workflows_tree():
     File("job-specs/docker_jobs.yml"),
     Header("Workflows"),
     Treegen(gen_build_workflows_tree, 0),
-    File("workflows/workflows-scheduled-ci.yml"),
     File("workflows/workflows-ecr-gc.yml"),
     File("workflows/workflows-promote.yml"),
 ]

diff --git a/.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml b/.circleci/verbatim-sources/job-specs/pytorch-job-specs.yml
@@ -190,8 +190,6 @@ jobs:
           if [[ ${BUILD_ENVIRONMENT} == *"multigpu"* ]]; then
             echo ".jenkins/pytorch/multigpu-test.sh" >> docker_commands.sh
           elif [[ ${BUILD_ENVIRONMENT} == *onnx* ]]; then
-            echo "pip install click mock tabulate networkx==2.0" >> docker_commands.sh
-            echo "pip -q install --user \"file:///var/lib/jenkins/workspace/third_party/onnx#egg=onnx\"" >> docker_commands.sh
             echo ".jenkins/caffe2/test.sh" >> docker_commands.sh
           else
             echo ".jenkins/pytorch/test.sh" >> docker_commands.sh

diff --git a/.circleci/verbatim-sources/workflows/workflows-scheduled-ci.yml b/.circleci/verbatim-sources/workflows/workflows-scheduled-ci.yml