Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 11 additions & 58 deletions ci/travis/format.sh
Original file line number Diff line number Diff line change
@@ -1,15 +1,12 @@
#!/usr/bin/env bash
# YAPF + Clang formatter (if installed). This script formats all changed files from the last mergebase.
# Black + Clang formatter (if installed). This script formats all changed files from the last mergebase.
# You are encouraged to run this locally before pushing changes for review.

# Cause the script to exit if a single command fails
set -euo pipefail

BLACK_IS_ENABLED=true

FLAKE8_VERSION_REQUIRED="3.9.1"
BLACK_VERSION_REQUIRED="21.12b0"
YAPF_VERSION_REQUIRED="0.23.0"
SHELLCHECK_VERSION_REQUIRED="0.7.1"
MYPY_VERSION_REQUIRED="0.782"

Expand All @@ -19,9 +16,6 @@ check_command_exist() {
black)
VERSION=$BLACK_VERSION_REQUIRED
;;
yapf)
VERSION=$YAPF_VERSION_REQUIRED
;;
flake8)
VERSION=$FLAKE8_VERSION_REQUIRED
;;
Expand All @@ -41,11 +35,7 @@ check_command_exist() {
fi
}

if [ "$BLACK_IS_ENABLED" = true ]; then
check_command_exist black
else
check_command_exist yapf
fi
check_command_exist black
check_command_exist flake8
check_command_exist mypy

Expand All @@ -56,11 +46,7 @@ ROOT="$(git rev-parse --show-toplevel)"
builtin cd "$ROOT" || exit 1

FLAKE8_VERSION=$(flake8 --version | head -n 1 | awk '{print $1}')
if [ "$BLACK_IS_ENABLED" = true ]; then
BLACK_VERSION=$(black --version | awk '{print $2}')
else
YAPF_VERSION=$(yapf --version | awk '{print $2}')
fi
BLACK_VERSION=$(black --version | awk '{print $2}')
SHELLCHECK_VERSION=$(shellcheck --version | awk '/^version:/ {print $2}')
MYPY_VERSION=$(mypy --version | awk '{print $2}')
GOOGLE_JAVA_FORMAT_JAR=/tmp/google-java-format-1.7-all-deps.jar
Expand All @@ -73,11 +59,7 @@ tool_version_check() {
}

tool_version_check "flake8" "$FLAKE8_VERSION" "$FLAKE8_VERSION_REQUIRED"
if [ "$BLACK_IS_ENABLED" = true ]; then
tool_version_check "black" "$BLACK_VERSION" "$BLACK_VERSION_REQUIRED"
else
tool_version_check "yapf" "$YAPF_VERSION" "$YAPF_VERSION_REQUIRED"
fi
tool_version_check "black" "$BLACK_VERSION" "$BLACK_VERSION_REQUIRED"
tool_version_check "shellcheck" "$SHELLCHECK_VERSION" "$SHELLCHECK_VERSION_REQUIRED"
tool_version_check "mypy" "$MYPY_VERSION" "$MYPY_VERSION_REQUIRED"

Expand Down Expand Up @@ -111,12 +93,6 @@ SHELLCHECK_FLAGS=(
--exclude=2207 # "Prefer mapfile or read -a to split command output (or quote to avoid splitting)." -- these aren't compatible with macOS's old Bash
)

YAPF_FLAGS=(
'--style' "$ROOT/.style.yapf"
'--recursive'
'--parallel'
)

# TODO(dmitri): When more of the codebase is typed properly, the mypy flags
# should be set to do a more stringent check.
MYPY_FLAGS=(
Expand Down Expand Up @@ -144,14 +120,6 @@ BLACK_EXCLUDES=(
'--extend-exclude' 'python/ray/_private/thirdparty/*'
)

YAPF_EXCLUDES=(
'--exclude' 'python/ray/cloudpickle/*'
'--exclude' 'python/build/*'
'--exclude' 'python/ray/core/src/ray/gcs/*'
'--exclude' 'python/ray/thirdparty_files/*'
'--exclude' 'python/ray/_private/thirdparty/*'
)

GIT_LS_EXCLUDES=(
':(exclude)python/ray/cloudpickle/'
)
Expand Down Expand Up @@ -220,11 +188,7 @@ format_files() {
done

if [ 0 -lt "${#python_files[@]}" ]; then
if [ "$BLACK_IS_ENABLED" = true ]; then
black "${python_files[@]}"
else
yapf --in-place "${YAPF_FLAGS[@]}" -- "${python_files[@]}"
fi
black "${python_files[@]}"
fi

if shellcheck --shell=sh --format=diff - < /dev/null; then
Expand All @@ -243,15 +207,9 @@ format_all_scripts() {
command -v flake8 &> /dev/null;
HAS_FLAKE8=$?

if [ "$BLACK_IS_ENABLED" = true ]; then
echo "$(date)" "Black...."
git ls-files -- '*.py' "${GIT_LS_EXCLUDES[@]}" | xargs -P 10 \
black "${BLACK_EXCLUDES[@]}"
else
echo "$(date)" "YAPF...."
git ls-files -- '*.py' "${GIT_LS_EXCLUDES[@]}" | xargs -P 10 \
yapf --in-place "${YAPF_EXCLUDES[@]}" "${YAPF_FLAGS[@]}"
fi
echo "$(date)" "Black...."
git ls-files -- '*.py' "${GIT_LS_EXCLUDES[@]}" | xargs -P 10 \
black "${BLACK_EXCLUDES[@]}"
echo "$(date)" "MYPY...."
mypy_on_each "${MYPY_FILES[@]}"
if [ $HAS_FLAKE8 ]; then
Expand Down Expand Up @@ -300,20 +258,15 @@ format_all() {
format_changed() {
# The `if` guard ensures that the list of filenames is not empty, which
# could cause the formatter to receive 0 positional arguments, making
# Black error and yapf hang waiting for STDIN.
# Black error.
#
# `diff-filter=ACRM` and $MERGEBASE is to ensure we only format files that
# exist on both branches.
MERGEBASE="$(git merge-base upstream/master HEAD)"

if ! git diff --diff-filter=ACRM --quiet --exit-code "$MERGEBASE" -- '*.py' &>/dev/null; then
if [ "$BLACK_IS_ENABLED" = true ]; then
git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.py' | xargs -P 5 \
black "${BLACK_EXCLUDES[@]}"
else
git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.py' | xargs -P 5 \
yapf --in-place "${YAPF_EXCLUDES[@]}" "${YAPF_FLAGS[@]}"
fi
git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.py' | xargs -P 5 \
black "${BLACK_EXCLUDES[@]}"
if which flake8 >/dev/null; then
git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.py' | xargs -P 5 \
flake8 --config=.flake8
Expand Down