Merge pull request #1887 from csarofeen/upstream_merge_0803

Upstream merge 0803

.bazelrc

@@ -31,6 +31,10 @@ build --copt=-isystem --copt=bazel-out/k8-fastbuild-cpu-only/bin
 # rules_cuda configuration
 build:cpu-only --@rules_cuda//cuda:enable_cuda=False
 
+# Definition of --config=shell
+# interactive shell immediately before execution
+build:shell --run_under="//tools/bazel_tools:shellwrap"
+
 # Disable all warnings for external repositories. We don't care about
 # their warnings.
 build --per_file_copt=^external/@-w
@@ -103,6 +107,9 @@ build --per_file_copt='//:aten/src/ATen/RegisterNestedTensorCPU\.cpp$'@-Wno-erro
 build --per_file_copt='//:aten/src/ATen/RegisterQuantizedCPU\.cpp$'@-Wno-error=unused-function
 build --per_file_copt='//:aten/src/ATen/RegisterSparseCPU\.cpp$'@-Wno-error=unused-function
 build --per_file_copt='//:aten/src/ATen/RegisterSparseCsrCPU\.cpp$'@-Wno-error=unused-function
+build --per_file_copt='//:aten/src/ATen/RegisterNestedTensorMeta\.cpp$'@-Wno-error=unused-function
+build --per_file_copt='//:aten/src/ATen/RegisterSparseMeta\.cpp$'@-Wno-error=unused-function
+build --per_file_copt='//:aten/src/ATen/RegisterQuantizedMeta\.cpp$'@-Wno-error=unused-function
 build --per_file_copt='//:aten/src/ATen/RegisterZeroTensor\.cpp$'@-Wno-error=unused-function
 build --per_file_copt='//:torch/csrc/lazy/generated/RegisterAutogradLazy\.cpp$'@-Wno-error=unused-function
 build --per_file_copt='//:torch/csrc/lazy/generated/RegisterLazy\.cpp$'@-Wno-error=unused-function

.buckconfig.oss

@@ -3,16 +3,23 @@
 
 [buildfile]
  name = BUCK.oss
+ includes = //tools/build_defs/select.bzl
 
 [repositories]
  bazel_skylib = third_party/bazel-skylib/
+ ovr_config = .
 
 [download]
  in_build = true
 
 [cxx]
  cxxflags = -std=c++17
  should_remap_host_platform = true
+ cpp = /usr/bin/clang
+ cc = /usr/bin/clang
+ cxx = /usr/bin/clang++
+ cxxpp = /usr/bin/clang++
+ ld = /usr/bin/clang++
 
 [project]
  default_flavors_mode=all

.circleci/cimodel/data/pytorch_build_data.py

@@ -75,6 +75,7 @@ def child_constructor(self):
  "vulkan": VulkanConfigNode,
  "parallel_tbb": ParallelTBBConfigNode,
  "crossref": CrossRefConfigNode,
+ "dynamo": DynamoConfigNode,
  "parallel_native": ParallelNativeConfigNode,
  "onnx": ONNXConfigNode,
  "libtorch": LibTorchConfigNode,
@@ -179,6 +180,14 @@ def child_constructor(self):
  return ImportantConfigNode
 
 
+class DynamoConfigNode(TreeConfigNode):
+ def init2(self, node_name):
+ self.props["is_dynamo"] = node_name
+
+ def child_constructor(self):
+ return ImportantConfigNode
+
+
 class ParallelNativeConfigNode(TreeConfigNode):
  def modify_label(self, label):
  return "PARALLELNATIVE=" + str(label)

.circleci/cimodel/data/pytorch_build_definitions.py

@@ -240,6 +240,7 @@ def instantiate_configs(only_slow_gradcheck):
  is_xla = fc.find_prop("is_xla") or False
  is_asan = fc.find_prop("is_asan") or False
  is_crossref = fc.find_prop("is_crossref") or False
+ is_dynamo = fc.find_prop("is_dynamo") or False
  is_onnx = fc.find_prop("is_onnx") or False
  is_pure_torch = fc.find_prop("is_pure_torch") or False
  is_vulkan = fc.find_prop("is_vulkan") or False
@@ -286,6 +287,9 @@ def instantiate_configs(only_slow_gradcheck):
  if is_crossref:
  parms_list_ignored_for_docker_image.append("crossref")
 
+ if is_dynamo:
+ parms_list_ignored_for_docker_image.append("dynamo")
+
  if is_onnx:
  parms_list.append("onnx")
  python_version = fc.find_prop("pyver")

.circleci/docker/android/build.gradle

@@ -53,7 +53,7 @@ dependencies {
  implementation 'androidx.appcompat:appcompat:1.0.0'
  implementation 'com.facebook.fbjni:fbjni-java-only:0.2.2'
  implementation 'com.google.code.findbugs:jsr305:3.0.1'
- implementation 'com.facebook.soloader:nativeloader:0.10.1'
+ implementation 'com.facebook.soloader:nativeloader:0.10.4'
 
  implementation 'junit:junit:' + rootProject.junitVersion
  implementation 'androidx.test:core:' + rootProject.coreVersion

.circleci/docker/build.sh

@@ -139,9 +139,19 @@ case "$image" in
  KATEX=yes
  ;;
  pytorch-linux-bionic-cuda11.6-cudnn8-py3-gcc7)
- CUDA_VERSION=11.6.0
+ CUDA_VERSION=11.6.2
  CUDNN_VERSION=8
- ANACONDA_PYTHON_VERSION=3.7
+ ANACONDA_PYTHON_VERSION=3.10
+ GCC_VERSION=7
+ PROTOBUF=yes
+ DB=yes
+ VISION=yes
+ KATEX=yes
+ ;;
+ pytorch-linux-bionic-cuda11.7-cudnn8-py3-gcc7)
+ CUDA_VERSION=11.7.0
+ CUDNN_VERSION=8
+ ANACONDA_PYTHON_VERSION=3.10
  GCC_VERSION=7
  PROTOBUF=yes
  DB=yes
@@ -176,6 +186,13 @@ case "$image" in
  DB=yes
  VISION=yes
  ;;
+ pytorch-linux-focal-py3-clang10-onnx)
+ ANACONDA_PYTHON_VERSION=3.7
+ CLANG_VERSION=10
+ PROTOBUF=yes
+ DB=yes
+ VISION=yes
+ ;;
  pytorch-linux-xenial-py3-clang5-android-ndk-r19c)
  ANACONDA_PYTHON_VERSION=3.7
  CLANG_VERSION=5.0
@@ -227,21 +244,21 @@ case "$image" in
  DB=yes
  VISION=yes
  ;;
- pytorch-linux-bionic-rocm5.0-py3.7)
+ pytorch-linux-focal-rocm5.1-py3.7)
  ANACONDA_PYTHON_VERSION=3.7
  GCC_VERSION=9
  PROTOBUF=yes
  DB=yes
  VISION=yes
- ROCM_VERSION=5.0
+ ROCM_VERSION=5.1.1
  ;;
- pytorch-linux-bionic-rocm5.1-py3.7)
+ pytorch-linux-focal-rocm5.2-py3.7)
  ANACONDA_PYTHON_VERSION=3.7
  GCC_VERSION=9
  PROTOBUF=yes
  DB=yes
  VISION=yes
- ROCM_VERSION=5.1.1
+ ROCM_VERSION=5.2
  ;;
  pytorch-linux-focal-py3.7-gcc7)
  ANACONDA_PYTHON_VERSION=3.7
@@ -261,6 +278,15 @@ case "$image" in
  DB=yes
  VISION=yes
  ;;
+ pytorch-linux-jammy-cuda11.7-cudnn8-py3.8-clang12)
+ ANACONDA_PYTHON_VERSION=3.8
+ CUDA_VERSION=11.7
+ CUDNN_VERSION=8
+ CLANG_VERSION=12
+ PROTOBUF=yes
+ DB=yes
+ VISION=yes
+ ;;
  *)
  # Catch-all for builds that are not hardcoded.
  PROTOBUF=yes

.circleci/docker/build_docker.sh

@@ -18,6 +18,7 @@ tag="${DOCKER_TAG}"
 
 registry="308535385114.dkr.ecr.us-east-1.amazonaws.com"
 image="${registry}/pytorch/${IMAGE_NAME}"
+ghcr_image="ghcr.io/pytorch/ci-image"
 
 login() {
  aws ecr get-authorization-token --region us-east-1 --output text --query 'authorizationData[].authorizationToken' |
@@ -48,7 +49,19 @@ fi
 
 # Only push if `DOCKER_SKIP_PUSH` = false
 if [ "${DOCKER_SKIP_PUSH:-true}" = "false" ]; then
- docker push "${image}:${tag}"
+ # Only push if docker image doesn't exist already.
+ # ECR image tags are immutable so this will avoid pushing if only just testing if the docker jobs work
+ # NOTE: The only workflow that should push these images should be the docker-builds.yml workflow
+ if ! docker manifest inspect "${image}:${tag}" >/dev/null 2>/dev/null; then
+ docker push "${image}:${tag}"
+ fi
+
+ if [ "${PUSH_GHCR_IMAGE:-}" = "true" ]; then
+ # Push docker image to the ghcr.io
+ echo $GHCR_PAT | docker login ghcr.io -u pytorch --password-stdin
+ docker tag "${image}:${tag}" "${ghcr_image}:${IMAGE_NAME}-${tag}"
+ docker push "${ghcr_image}:${IMAGE_NAME}-${tag}"
+ fi
 fi
 
 if [ -z "${DOCKER_SKIP_S3_UPLOAD:-}" ]; then

.circleci/docker/centos-rocm/Dockerfile

@@ -12,7 +12,7 @@ ENV PYTORCH_ROCM_ARCH ${PYTORCH_ROCM_ARCH}
 
 # Install common dependencies (so that this step can be cached separately)
 ARG EC2
-ADD ./common/install_base.sh install_base.sh
+COPY ./common/install_base.sh install_base.sh
 RUN bash ./install_base.sh && rm install_base.sh
 
 # Update CentOS git version
@@ -23,52 +23,52 @@ RUN yum install -y git
 
 # Install devtoolset
 ARG DEVTOOLSET_VERSION
-ADD ./common/install_devtoolset.sh install_devtoolset.sh
+COPY ./common/install_devtoolset.sh install_devtoolset.sh
 RUN bash ./install_devtoolset.sh && rm install_devtoolset.sh
 ENV BASH_ENV "/etc/profile"
 
 # (optional) Install non-default glibc version
 ARG GLIBC_VERSION
-ADD ./common/install_glibc.sh install_glibc.sh
+COPY ./common/install_glibc.sh install_glibc.sh
 RUN if [ -n "${GLIBC_VERSION}" ]; then bash ./install_glibc.sh; fi
 RUN rm install_glibc.sh
 
 # Install user
-ADD ./common/install_user.sh install_user.sh
+COPY ./common/install_user.sh install_user.sh
 RUN bash ./install_user.sh && rm install_user.sh
 
 # Install conda and other packages (e.g., numpy, pytest)
 ENV PATH /opt/conda/bin:$PATH
 ARG ANACONDA_PYTHON_VERSION
-ADD requirements-ci.txt /opt/conda/requirements-ci.txt
-ADD ./common/install_conda.sh install_conda.sh
+COPY requirements-ci.txt /opt/conda/requirements-ci.txt
+COPY ./common/install_conda.sh install_conda.sh
 RUN bash ./install_conda.sh && rm install_conda.sh
 RUN rm /opt/conda/requirements-ci.txt
 
 # (optional) Install protobuf for ONNX
 ARG PROTOBUF
-ADD ./common/install_protobuf.sh install_protobuf.sh
+COPY ./common/install_protobuf.sh install_protobuf.sh
 RUN if [ -n "${PROTOBUF}" ]; then bash ./install_protobuf.sh; fi
 RUN rm install_protobuf.sh
 ENV INSTALLED_PROTOBUF ${PROTOBUF}
 
 # (optional) Install database packages like LMDB and LevelDB
 ARG DB
-ADD ./common/install_db.sh install_db.sh
+COPY ./common/install_db.sh install_db.sh
 RUN if [ -n "${DB}" ]; then bash ./install_db.sh; fi
 RUN rm install_db.sh
 ENV INSTALLED_DB ${DB}
 
 # (optional) Install vision packages like OpenCV and ffmpeg
 ARG VISION
-ADD ./common/install_vision.sh install_vision.sh
+COPY ./common/install_vision.sh install_vision.sh
 RUN if [ -n "${VISION}" ]; then bash ./install_vision.sh; fi
 RUN rm install_vision.sh
 ENV INSTALLED_VISION ${VISION}
 
 # Install rocm
 ARG ROCM_VERSION
-ADD ./common/install_rocm.sh install_rocm.sh
+COPY ./common/install_rocm.sh install_rocm.sh
 RUN bash ./install_rocm.sh
 RUN rm install_rocm.sh
 ENV PATH /opt/rocm/bin:$PATH
@@ -82,18 +82,18 @@ ENV LC_ALL en_US.utf8
 
 # (optional) Install non-default CMake version
 ARG CMAKE_VERSION
-ADD ./common/install_cmake.sh install_cmake.sh
+COPY ./common/install_cmake.sh install_cmake.sh
 RUN if [ -n "${CMAKE_VERSION}" ]; then bash ./install_cmake.sh; fi
 RUN rm install_cmake.sh
 
 # (optional) Install non-default Ninja version
 ARG NINJA_VERSION
-ADD ./common/install_ninja.sh install_ninja.sh
+COPY ./common/install_ninja.sh install_ninja.sh
 RUN if [ -n "${NINJA_VERSION}" ]; then bash ./install_ninja.sh; fi
 RUN rm install_ninja.sh
 
 # Install ccache/sccache (do this last, so we get priority in PATH)
-ADD ./common/install_cache.sh install_cache.sh
+COPY ./common/install_cache.sh install_cache.sh
 ENV PATH /opt/cache/bin:$PATH
 RUN bash ./install_cache.sh && rm install_cache.sh
 

.circleci/docker/common/install_base.sh

@@ -24,9 +24,11 @@ install_ubuntu() {
  fi
 
  if [[ "$CLANG_VERSION" == 12 ]]; then
- libomp_dev="libomp-12-dev"
+ maybe_libomp_dev="libomp-12-dev"
+ elif [[ "$CLANG_VERSION" == 10 ]]; then
+ maybe_libomp_dev="libomp-10-dev"
  else
- libomp_dev=""
+ maybe_libomp_dev=""
  fi
 
  # TODO: Remove this once nvidia package repos are back online
@@ -60,11 +62,12 @@ install_ubuntu() {
  libjpeg-dev \
  libasound2-dev \
  libsndfile-dev \
- ${libomp_dev} \
+ ${maybe_libomp_dev} \
  software-properties-common \
  wget \
  sudo \
- vim
+ vim \
+ jq
 
  # Should resolve issues related to various apt package repository cert issues
  # see: https://github.com/pytorch/pytorch/issues/65931

.circleci/docker/common/install_clang.sh

@@ -13,6 +13,9 @@ if [ -n "$CLANG_VERSION" ]; then
  sudo apt-get install -y --no-install-recommends gpg-agent
  wget --no-check-certificate -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
  apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-${CLANG_VERSION} main"
+ elif [[ $UBUNTU_VERSION == 22.04 ]]; then
+ # work around ubuntu apt-get conflicts
+ sudo apt-get -y -f install
  fi
 
  sudo apt-get update

.circleci/docker/common/install_conda.sh

@@ -73,19 +73,21 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
  }
 
  # Install PyTorch conda deps, as per https://github.com/pytorch/pytorch README
- # DO NOT install cmake here as it would install a version newer than 3.10, but
- # we want to pin to version 3.10.
- if [ "$ANACONDA_PYTHON_VERSION" = "3.9" ]; then
+ # DO NOT install cmake here as it would install a version newer than 3.13, but
+ # we want to pin to version 3.13.
+ CONDA_COMMON_DEPS="astunparse pyyaml mkl=2022.0.1 mkl-include=2022.0.1 setuptools cffi future six"
+ if [ "$ANACONDA_PYTHON_VERSION" = "3.10" ]; then
  # Install llvm-8 as it is required to compile llvmlite-0.30.0 from source
- conda_install numpy=1.19.2 astunparse pyyaml mkl mkl-include setuptools cffi future six llvmdev=8.0.0
+ conda_install numpy=1.21.2 ${CONDA_COMMON_DEPS} llvmdev=8.0.0
+ elif [ "$ANACONDA_PYTHON_VERSION" = "3.9" ]; then
+ # Install llvm-8 as it is required to compile llvmlite-0.30.0 from source
+ conda_install numpy=1.19.2 ${CONDA_COMMON_DEPS} llvmdev=8.0.0
  elif [ "$ANACONDA_PYTHON_VERSION" = "3.8" ]; then
  # Install llvm-8 as it is required to compile llvmlite-0.30.0 from source
- conda_install numpy=1.18.5 astunparse pyyaml mkl mkl-include setuptools cffi future six llvmdev=8.0.0
- elif [ "$ANACONDA_PYTHON_VERSION" = "3.7" ]; then
- # DO NOT install dataclasses if installing python-3.7, since its part of python-3.7 core packages
- conda_install numpy=1.18.5 astunparse pyyaml mkl mkl-include setuptools cffi future six typing_extensions
+ conda_install numpy=1.18.5 ${CONDA_COMMON_DEPS} llvmdev=8.0.0
  else
- conda_install numpy=1.18.5 astunparse pyyaml mkl mkl-include setuptools cffi future six dataclasses typing_extensions
+ # Install `typing_extensions` for 3.7
+ conda_install numpy=1.18.5 ${CONDA_COMMON_DEPS} typing_extensions
  fi
 
  # Magma package names are concatenation of CUDA major and minor ignoring revision

.circleci/docker/common/install_rocm.sh

@@ -35,7 +35,7 @@ ver() {
 }
 
 # Map ROCm version to AMDGPU version
-declare -A AMDGPU_VERSIONS=( ["4.5.2"]="21.40.2" ["5.0"]="21.50" ["5.1.1"]="22.10.1" )
+declare -A AMDGPU_VERSIONS=( ["5.0"]="21.50" ["5.1.1"]="22.10.1" ["5.2"]="22.20" )
 
 install_ubuntu() {
  apt-get update