From b320be417396c6fb15ac880c658e911233e5b16f Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Thu, 28 May 2020 02:43:44 +0000 Subject: [PATCH 1/5] Separate style check from unit test --- .circleci/config.yml | 34 +++++++++++++++++++ .circleci/config.yml.in | 31 +++++++++++++++++ .circleci/regenerate.py | 10 +++++- .../unittest/linux/scripts/environment.yml | 1 + .../linux/scripts/run_style_checks.sh | 32 +++++++++++++++++ .circleci/unittest/linux/scripts/run_test.sh | 1 - 6 files changed, 107 insertions(+), 2 deletions(-) create mode 100755 .circleci/unittest/linux/scripts/run_style_checks.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index a1658f6bcc..e05bef612a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -471,6 +471,37 @@ jobs: - store_test_results: path: test-results + stylecheck: + <<: *binary_common + docker: + - image: "pytorch/torchaudio_unittest_base:manylinux" + resource_class: medium + steps: + - checkout + - run: + name: Generate cache key + # This will refresh cache on Sundays, nightly build should generate new cache. + command: echo "$(date +"%Y-%U")" > .circleci-weekly + - restore_cache: + + keys: + - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + - run: + name: Setup + command: .circleci/unittest/linux/scripts/setup_env.sh + - save_cache: + + key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + paths: + - conda + - env + - third_party/build + - run: + name: Run style check + command: .circleci/unittest/linux/scripts/run_style_checks.sh + workflows: build: jobs: @@ -534,6 +565,9 @@ workflows: - unittest_linux_cpu: name: unittest_linux_cpu_py3.6 python_version: '3.6' + - stylecheck: + name: stylecheck_py3.6 + python_version: '3.6' - unittest_linux_cpu: name: unittest_linux_cpu_py3.7 python_version: '3.7' diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index bb2a67d7a0..eb3ed50bf3 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -471,6 +471,37 @@ jobs: - store_test_results: path: test-results + stylecheck: + <<: *binary_common + docker: + - image: "pytorch/torchaudio_unittest_base:manylinux" + resource_class: medium + steps: + - checkout + - run: + name: Generate cache key + # This will refresh cache on Sundays, nightly build should generate new cache. + command: echo "$(date +"%Y-%U")" > .circleci-weekly + - restore_cache: + {% raw %} + keys: + - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + {% endraw %} + - run: + name: Setup + command: .circleci/unittest/linux/scripts/setup_env.sh + - save_cache: + {% raw %} + key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + {% endraw %} + paths: + - conda + - env + - third_party/build + - run: + name: Run style check + command: .circleci/unittest/linux/scripts/run_style_checks.sh + workflows: build: jobs: diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 042e4c1cb3..8626daf3f3 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -114,7 +114,7 @@ def unittest_workflows(indentation=6): jobs = [] for os_type in ["linux", "windows"]: for device_type in ["cpu", "gpu"]: - for python_version in PYTHON_VERSIONS: + for i, python_version in enumerate(PYTHON_VERSIONS): job = { "name": f"unittest_{os_type}_{device_type}_py{python_version}", "python_version": python_version, @@ -123,6 +123,14 @@ def unittest_workflows(indentation=6): if device_type == 'gpu': job['filters'] = gen_filter_branch_tree('master') jobs.append({f"unittest_{os_type}_{device_type}": job}) + + if i == 0 and os_type == "linux" and device_type == "cpu": + jobs.append({ + f"stylecheck": { + "name": f"stylecheck_py{python_version}", + "python_version": python_version, + } + }) return indent(indentation, jobs) diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml index 26f1bf0f7e..e8a9a289e7 100644 --- a/.circleci/unittest/linux/scripts/environment.yml +++ b/.circleci/unittest/linux/scripts/environment.yml @@ -10,5 +10,6 @@ dependencies: - librosa - pip - pip: + - clang-format - kaldi-io - scipy diff --git a/.circleci/unittest/linux/scripts/run_style_checks.sh b/.circleci/unittest/linux/scripts/run_style_checks.sh new file mode 100755 index 0000000000..70b83e19f5 --- /dev/null +++ b/.circleci/unittest/linux/scripts/run_style_checks.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -u + +eval "$(./conda/bin/conda shell.bash hook)" +conda activate ./env + +# We want to run all the style checks even if one of them fail. + +exit_status=0 + +printf "\x1b[34mRunning flake8: " +flake8 --version +printf "\x1b[0m\n" +flake8 torchaudio test build_tools/setup_helpers +status=$? +exit_status="$((exit_status+status))" +if [ "${status}" -ne 0 ]; then + printf "\x1b[31mflake8 failed. Check the format of Python files.\x1b[0m\n" +fi + +printf "\x1b[34mRunning clang-format: " +clang-format --version +printf "\x1b[0m\n" +git-clang-format origin/master +git diff --exit-code +status=$? +exit_status="$((exit_status+status))" +if [ "${status}" -ne 0 ]; then + printf "\x1b[31mC++ files are not formatted. Please use git-clang-format to format CPP files.\x1b[0m\n" +fi +exit $exit_status diff --git a/.circleci/unittest/linux/scripts/run_test.sh b/.circleci/unittest/linux/scripts/run_test.sh index 7862bb05e3..700eb7ad64 100755 --- a/.circleci/unittest/linux/scripts/run_test.sh +++ b/.circleci/unittest/linux/scripts/run_test.sh @@ -7,4 +7,3 @@ conda activate ./env python -m torch.utils.collect_env pytest --cov=torchaudio --junitxml=test-results/junit.xml -v --durations 20 test -flake8 torchaudio test build_tools/setup_helpers From 88d90185cefe294b9a662129a05c2e26a9844fe4 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Wed, 27 May 2020 20:02:11 -0700 Subject: [PATCH 2/5] TEST flake8 fail --- torchaudio/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/torchaudio/__init__.py b/torchaudio/__init__.py index a08eaf6964..dd30fa9922 100644 --- a/torchaudio/__init__.py +++ b/torchaudio/__init__.py @@ -27,6 +27,9 @@ pass + + + def load(filepath: Union[str, Path], out: Optional[Tensor] = None, normalization: Union[bool, float, Callable] = True, From 93517d1ccba650a6ccd813268fd73a4fd63b252c Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Thu, 28 May 2020 03:06:51 +0000 Subject: [PATCH 3/5] TEST clang-format fail --- torchaudio/csrc/sox.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/torchaudio/csrc/sox.cpp b/torchaudio/csrc/sox.cpp index 3ae81bef19..bdc93056a6 100644 --- a/torchaudio/csrc/sox.cpp +++ b/torchaudio/csrc/sox.cpp @@ -6,7 +6,10 @@ #include namespace torch { -namespace audio { + + + + namespace audio { namespace { /// Helper struct to safely close the sox_format_t descriptor. struct SoxDescriptor { From 79090e2cc331a231d1500c6267b47625c0476b06 Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Thu, 28 May 2020 03:11:35 +0000 Subject: [PATCH 4/5] Revert "TEST flake8 fail" This reverts commit 88d90185cefe294b9a662129a05c2e26a9844fe4. --- torchaudio/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/torchaudio/__init__.py b/torchaudio/__init__.py index dd30fa9922..a08eaf6964 100644 --- a/torchaudio/__init__.py +++ b/torchaudio/__init__.py @@ -27,9 +27,6 @@ pass - - - def load(filepath: Union[str, Path], out: Optional[Tensor] = None, normalization: Union[bool, float, Callable] = True, From 63fd02d09a3776e6b9f99673a19db8d9d5c49daf Mon Sep 17 00:00:00 2001 From: moto <855818+mthrok@users.noreply.github.com> Date: Thu, 28 May 2020 03:11:40 +0000 Subject: [PATCH 5/5] Revert "TEST clang-format fail" This reverts commit 93517d1ccba650a6ccd813268fd73a4fd63b252c. --- torchaudio/csrc/sox.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/torchaudio/csrc/sox.cpp b/torchaudio/csrc/sox.cpp index bdc93056a6..3ae81bef19 100644 --- a/torchaudio/csrc/sox.cpp +++ b/torchaudio/csrc/sox.cpp @@ -6,10 +6,7 @@ #include namespace torch { - - - - namespace audio { +namespace audio { namespace { /// Helper struct to safely close the sox_format_t descriptor. struct SoxDescriptor {