diff --git a/.flake8 b/.flake8 index db18ef3e9..50aec90b6 100644 --- a/.flake8 +++ b/.flake8 @@ -9,13 +9,14 @@ exclude = python/.eggs/ python/ray/_private/thirdparty/* python/ray/workflow/tests/mock_server.py -max-line-length = 79 +max-line-length = 88 inline-quotes = " ignore = C408 E121 E123 E126 + E203 E226 E24 E704 diff --git a/.gitpod/Dockerfile b/.gitpod/Dockerfile index 441c6eb9d..ad2f00d44 100644 --- a/.gitpod/Dockerfile +++ b/.gitpod/Dockerfile @@ -15,7 +15,7 @@ RUN set -x; apt update \ && mv bazel.gpg /etc/apt/trusted.gpg.d/ \ && echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list \ && apt update && apt install bazel-3.7.2 -y \ - && pip3 install cython==0.29.26 pytest pandas tree tabulate pexpect sklearn joblib yapf==0.23.0 flake8==3.9.1 mypy==0.782 flake8-quotes flake8-bugbear==21.9.2 setproctitle==1.1.10 psutil \ + && pip3 install cython==0.29.26 pytest pandas tree tabulate pexpect sklearn joblib black==21.12b0 yapf==0.23.0 flake8==3.9.1 mypy==0.782 flake8-quotes flake8-bugbear==21.9.2 setproctitle==1.1.10 psutil \ && python3 -c 'print("startup --output_base=/workspace/ray/.bazel-cache\nstartup --host_jvm_args=-Xmx1800m\nbuild --jobs=6")' > /etc/bazel.bazelrc RUN update-alternatives --install /usr/local/bin/python python /usr/bin/python3 30 \ diff --git a/ci/travis/format.sh b/ci/travis/format.sh index 7dbf608d1..9b7638a57 100755 --- a/ci/travis/format.sh +++ b/ci/travis/format.sh @@ -5,7 +5,10 @@ # Cause the script to exit if a single command fails set -euo pipefail +BLACK_IS_ENABLED=false + FLAKE8_VERSION_REQUIRED="3.9.1" +BLACK_VERSION_REQUIRED="21.12b0" YAPF_VERSION_REQUIRED="0.23.0" SHELLCHECK_VERSION_REQUIRED="0.7.1" MYPY_VERSION_REQUIRED="0.782" @@ -13,6 +16,9 @@ MYPY_VERSION_REQUIRED="0.782" check_command_exist() { VERSION="" case "$1" in + black) + VERSION=$BLACK_VERSION_REQUIRED + ;; yapf) VERSION=$YAPF_VERSION_REQUIRED ;; @@ -35,7 +41,11 @@ check_command_exist() { fi } -check_command_exist yapf +if [ "$BLACK_IS_ENABLED" = true ]; then + check_command_exist black +else + check_command_exist yapf +fi check_command_exist flake8 check_command_exist mypy @@ -46,7 +56,11 @@ ROOT="$(git rev-parse --show-toplevel)" builtin cd "$ROOT" || exit 1 FLAKE8_VERSION=$(flake8 --version | head -n 1 | awk '{print $1}') -YAPF_VERSION=$(yapf --version | awk '{print $2}') +if [ "$BLACK_IS_ENABLED" = true ]; then + BLACK_VERSION=$(black --version | awk '{print $2}') +else + YAPF_VERSION=$(yapf --version | awk '{print $2}') +fi SHELLCHECK_VERSION=$(shellcheck --version | awk '/^version:/ {print $2}') MYPY_VERSION=$(mypy --version | awk '{print $2}') GOOGLE_JAVA_FORMAT_JAR=/tmp/google-java-format-1.7-all-deps.jar @@ -59,7 +73,11 @@ tool_version_check() { } tool_version_check "flake8" "$FLAKE8_VERSION" "$FLAKE8_VERSION_REQUIRED" -tool_version_check "yapf" "$YAPF_VERSION" "$YAPF_VERSION_REQUIRED" +if [ "$BLACK_IS_ENABLED" = true ]; then + tool_version_check "black" "$BLACK_VERSION" "$BLACK_VERSION_REQUIRED" +else + tool_version_check "yapf" "$YAPF_VERSION" "$YAPF_VERSION_REQUIRED" +fi tool_version_check "shellcheck" "$SHELLCHECK_VERSION" "$SHELLCHECK_VERSION_REQUIRED" tool_version_check "mypy" "$MYPY_VERSION" "$MYPY_VERSION_REQUIRED" @@ -117,6 +135,14 @@ MYPY_FILES=( 'ray_operator/operator_utils.py' ) +BLACK_EXCLUDES=( + '--extend-exclude' 'python/ray/cloudpickle/*' + '--extend-exclude' 'python/build/*' + '--extend-exclude' 'python/ray/core/src/ray/gcs/*' + '--extend-exclude' 'python/ray/thirdparty_files/*' + '--extend-exclude' 'python/ray/_private/thirdparty/*' +) + YAPF_EXCLUDES=( '--exclude' 'python/ray/cloudpickle/*' '--exclude' 'python/build/*' @@ -193,7 +219,11 @@ format_files() { done if [ 0 -lt "${#python_files[@]}" ]; then - yapf --in-place "${YAPF_FLAGS[@]}" -- "${python_files[@]}" + if [ "$BLACK_IS_ENABLED" = true ]; then + black "${python_files[@]}" + else + yapf --in-place "${YAPF_FLAGS[@]}" -- "${python_files[@]}" + fi fi if shellcheck --shell=sh --format=diff - < /dev/null; then @@ -212,9 +242,15 @@ format_all_scripts() { command -v flake8 &> /dev/null; HAS_FLAKE8=$? - echo "$(date)" "YAPF...." - git ls-files -- '*.py' "${GIT_LS_EXCLUDES[@]}" | xargs -P 10 \ - yapf --in-place "${YAPF_EXCLUDES[@]}" "${YAPF_FLAGS[@]}" + if [ "$BLACK_IS_ENABLED" = true ]; then + echo "$(date)" "Black...." + git ls-files -- '*.py' "${GIT_LS_EXCLUDES[@]}" | xargs -P 10 \ + black "${BLACK_EXCLUDES[@]}" + else + echo "$(date)" "YAPF...." + git ls-files -- '*.py' "${GIT_LS_EXCLUDES[@]}" | xargs -P 10 \ + yapf --in-place "${YAPF_EXCLUDES[@]}" "${YAPF_FLAGS[@]}" + fi echo "$(date)" "MYPY...." mypy_on_each "${MYPY_FILES[@]}" if [ $HAS_FLAKE8 ]; then @@ -262,16 +298,21 @@ format_all() { # for autoformat yet. format_changed() { # The `if` guard ensures that the list of filenames is not empty, which - # could cause yapf to receive 0 positional arguments, making it hang - # waiting for STDIN. + # could cause the formatter to receive 0 positional arguments, making + # Black error and yapf hang waiting for STDIN. # # `diff-filter=ACRM` and $MERGEBASE is to ensure we only format files that # exist on both branches. MERGEBASE="$(git merge-base upstream/master HEAD)" if ! git diff --diff-filter=ACRM --quiet --exit-code "$MERGEBASE" -- '*.py' &>/dev/null; then - git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.py' | xargs -P 5 \ - yapf --in-place "${YAPF_EXCLUDES[@]}" "${YAPF_FLAGS[@]}" + if [ "$BLACK_IS_ENABLED" = true ]; then + git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.py' | xargs -P 5 \ + black "${BLACK_EXCLUDES[@]}" + else + git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.py' | xargs -P 5 \ + yapf --in-place "${YAPF_EXCLUDES[@]}" "${YAPF_FLAGS[@]}" + fi if which flake8 >/dev/null; then git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.py' | xargs -P 5 \ flake8 --config=.flake8 diff --git a/python/ray/tune/requirements-dev.txt b/python/ray/tune/requirements-dev.txt index 26722c017..9a20329cd 100644 --- a/python/ray/tune/requirements-dev.txt +++ b/python/ray/tune/requirements-dev.txt @@ -6,4 +6,5 @@ pandas requests tabulate tensorflow +black==21.12b0 yapf==0.23.0 diff --git a/python/requirements_linters.txt b/python/requirements_linters.txt index 69f457fea..f36a8a970 100644 --- a/python/requirements_linters.txt +++ b/python/requirements_linters.txt @@ -3,4 +3,5 @@ flake8-comprehensions flake8-quotes==2.0.0 flake8-bugbear==21.9.2 mypy==0.782 +black==21.12b0 yapf==0.23.0