Add macOS unit test jobs

pytorch · Mar 14, 2021 · 8ef88bb · 8ef88bb
1 parent 735d5e7
commit 8ef88bb
Show file tree

Hide file tree

Showing 7 changed files with 196 additions and 15 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -177,13 +177,7 @@ jobs:
       - checkout
       - designate_upload_channel
       - run:
-          # Installing cmake with `brew install cmake` takes 30 mins, so we use binary distribution.
           command: |
-            curl -o cmake.tar.gz -L https://github.com/Kitware/CMake/releases/download/v3.17.2/cmake-3.17.2-Darwin-x86_64.tar.gz
-            tar -xzf cmake.tar.gz
-            cp cmake-3.17.2-Darwin-x86_64/CMake.app/Contents/bin/* /usr/local/bin/
-            cp -r cmake-3.17.2-Darwin-x86_64/CMake.app/Contents/share/* /usr/local/share/
-            export PATH="${PATH}:/usr/local/bin"
             curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
             sh conda.sh -b
             source $HOME/miniconda3/bin/activate
@@ -440,6 +434,89 @@ jobs:
       - store_test_results:
           path: test-results
 
+  cachesetup_macos:
+    <<: *binary_common
+    macos:
+      xcode: "9.4.1"
+    resource_class: 2xlarge+
+    steps:
+      - checkout
+      - designate_upload_channel
+      - load_conda_channel_flags
+      - generate_cachekey
+      - restore_cache:
+          keys:
+
+            - v1-macos-cache-index-{{ checksum ".cachekey" }}
+
+      - run:
+          name: Generate daily data Cache
+          no_output_timeout: 30m
+          command: |
+            if [ ! -f .data/cache_status_file.json ]; then
+              # Disable brew auto update which is very slow
+              HOMEBREW_NO_AUTO_UPDATE=1 brew install wget
+              .circleci/unittest/linux/scripts/setup_env.sh
+              .circleci/unittest/linux/scripts/install.sh
+              .circleci/unittest/linux/scripts/generate_cache.sh
+            fi
+            cat .data/cache_status_file.json
+      - save_cache:
+
+          key: v1-macos-dataset-{{ checksum ".cachekey" }}
+
+          paths:
+            - .data
+      - save_cache:
+
+          key: v1-macos-cache-index-{{ checksum ".cachekey" }}
+
+          paths:
+            - .data/cache_status_file.json
+
+  unittest_macos:
+    <<: *binary_common
+    macos:
+      xcode: "9.4.1"
+    resource_class: large
+    steps:
+      - checkout
+      - designate_upload_channel
+      - load_conda_channel_flags
+      - fetch_cachekey
+      - run:
+          name: Setup
+          command: |
+            # Disable brew auto update which is very slow
+            HOMEBREW_NO_AUTO_UPDATE=1 brew install wget
+            .circleci/unittest/linux/scripts/setup_env.sh
+      - run:
+          name: Install torchtext
+          command: .circleci/unittest/linux/scripts/install.sh
+      - restore_cache:
+          keys:
+
+            - v1-macos-dataset-vector-{{ checksum ".cachekey" }}
+            - v1-macos-dataset-{{ checksum ".cachekey" }}
+
+      - run:
+          name: Run tests
+          # Downloading embedding vector takes long time.
+          no_output_timeout: 30m
+          command: .circleci/unittest/linux/scripts/run_test.sh
+      - save_cache:
+
+          key: v1-macos-dataset-vector-{{ checksum ".cachekey" }}
+
+          paths:
+            - .vector_cache
+            - .data
+      - run:
+          name: Post process
+          command: .circleci/unittest/linux/scripts/post_process.sh
+      - store_test_results:
+          path: test-results
+
   cachesetup_windows:
     <<: *binary_common
     executor:
@@ -754,6 +831,29 @@ workflows:
           python_version: '3.9'
           requires:
           - cachesetup_windows_py_any
+      - cachesetup_macos:
+          name: cachesetup_macos_py_any
+          python_version: '3.6'
+      - unittest_macos:
+          name: unittest_macos_py3.6
+          python_version: '3.6'
+          requires:
+          - cachesetup_macos_py_any
+      - unittest_macos:
+          name: unittest_macos_py3.7
+          python_version: '3.7'
+          requires:
+          - cachesetup_macos_py_any
+      - unittest_macos:
+          name: unittest_macos_py3.8
+          python_version: '3.8'
+          requires:
+          - cachesetup_macos_py_any
+      - unittest_macos:
+          name: unittest_macos_py3.9
+          python_version: '3.9'
+          requires:
+          - cachesetup_macos_py_any
   nightly:
     jobs:
       - circleci_consistency:

diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in
@@ -177,13 +177,7 @@ jobs:
       - checkout
       - designate_upload_channel
       - run:
-          # Installing cmake with `brew install cmake` takes 30 mins, so we use binary distribution.
           command: |
-            curl -o cmake.tar.gz -L https://github.com/Kitware/CMake/releases/download/v3.17.2/cmake-3.17.2-Darwin-x86_64.tar.gz
-            tar -xzf cmake.tar.gz
-            cp cmake-3.17.2-Darwin-x86_64/CMake.app/Contents/bin/* /usr/local/bin/
-            cp -r cmake-3.17.2-Darwin-x86_64/CMake.app/Contents/share/* /usr/local/share/
-            export PATH="${PATH}:/usr/local/bin"
             curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
             sh conda.sh -b
             source $HOME/miniconda3/bin/activate
@@ -440,6 +434,89 @@ jobs:
       - store_test_results:
           path: test-results
 
+  cachesetup_macos:
+    <<: *binary_common
+    macos:
+      xcode: "9.4.1"
+    resource_class: 2xlarge+
+    steps:
+      - checkout
+      - designate_upload_channel
+      - load_conda_channel_flags
+      - generate_cachekey
+      - restore_cache:
+          keys:
+          {% raw %}
+            - v1-macos-cache-index-{{ checksum ".cachekey" }}
+          {% endraw %}
+      - run:
+          name: Generate daily data Cache
+          no_output_timeout: 30m
+          command: |
+            if [ ! -f .data/cache_status_file.json ]; then
+              # Disable brew auto update which is very slow
+              HOMEBREW_NO_AUTO_UPDATE=1 brew install wget
+              .circleci/unittest/linux/scripts/setup_env.sh
+              .circleci/unittest/linux/scripts/install.sh
+              .circleci/unittest/linux/scripts/generate_cache.sh
+            fi
+            cat .data/cache_status_file.json
+      - save_cache:
+          {% raw %}
+          key: v1-macos-dataset-{{ checksum ".cachekey" }}
+          {% endraw %}
+          paths:
+            - .data
+      - save_cache:
+          {% raw %}
+          key: v1-macos-cache-index-{{ checksum ".cachekey" }}
+          {% endraw %}
+          paths:
+            - .data/cache_status_file.json
+
+  unittest_macos:
+    <<: *binary_common
+    macos:
+      xcode: "9.4.1"
+    resource_class: large
+    steps:
+      - checkout
+      - designate_upload_channel
+      - load_conda_channel_flags
+      - fetch_cachekey
+      - run:
+          name: Setup
+          command: |
+            # Disable brew auto update which is very slow
+            HOMEBREW_NO_AUTO_UPDATE=1 brew install wget
+            .circleci/unittest/linux/scripts/setup_env.sh
+      - run:
+          name: Install torchtext
+          command: .circleci/unittest/linux/scripts/install.sh
+      - restore_cache:
+          keys:
+          {% raw %}
+            - v1-macos-dataset-vector-{{ checksum ".cachekey" }}
+            - v1-macos-dataset-{{ checksum ".cachekey" }}
+          {% endraw %}
+      - run:
+          name: Run tests
+          # Downloading embedding vector takes long time.
+          no_output_timeout: 30m
+          command: .circleci/unittest/linux/scripts/run_test.sh
+      - save_cache:
+          {% raw %}
+          key: v1-macos-dataset-vector-{{ checksum ".cachekey" }}
+          {% endraw %}
+          paths:
+            - .vector_cache
+            - .data
+      - run:
+          name: Post process
+          command: .circleci/unittest/linux/scripts/post_process.sh
+      - store_test_results:
+          path: test-results
+
   cachesetup_windows:
     <<: *binary_common
     executor:

diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py
@@ -152,7 +152,7 @@ def indent(indentation, data_list):
 
 def unittest_workflows(indentation=6):
     w = []
-    for os_type in ["linux", "windows"]:
+    for os_type in ["linux", "windows", "macos"]:
         w.append({
             f"cachesetup_{os_type}": {
                 "name": f"cachesetup_{os_type}_py_any",

diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml
@@ -6,6 +6,8 @@ dependencies:
   - codecov
   - pip
   - pip:
+    - cmake
+    - ninja
     - dataclasses
     - nltk
     - requests

diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh
@@ -22,7 +22,7 @@ esac
 # 1. Install conda at ./conda
 if [ ! -d "${conda_dir}" ]; then
     printf "* Installing conda\n"
-    wget -O miniconda.sh http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+    wget -O miniconda.sh "http://repo.continuum.io/miniconda/Miniconda3-latest-${os}-x86_64.sh"
     bash ./miniconda.sh -b -f -p "${conda_dir}"
 fi
 eval "$(${conda_dir}/bin/conda shell.bash hook)"

diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml
@@ -6,6 +6,8 @@ dependencies:
   - codecov
   - pip
   - pip:
+    - cmake
+    - ninja
     - dataclasses
     - nltk
     - requests

diff --git a/packaging/build_wheel.sh b/packaging/build_wheel.sh
@@ -8,7 +8,7 @@ export BUILD_TYPE="wheel"
 export NO_CUDA_PACKAGE=1
 setup_env 0.10.0
 setup_wheel_python
-pip_install numpy future
+pip_install numpy future cmake ninja
 setup_pip_pytorch_version
 git submodule update --init --recursive
 python setup.py clean
-Original file line number
+Diff line change
@@ Expand Up / @@ -6,6 +6,8 @@ dependencies: @@
       - codecov
       - pip
       - pip:
+        - cmake
+        - ninja
         - dataclasses
         - nltk
         - requests
@@ Expand Down @@