[CI] Add support for Black formatting (#21281)

.flake8

@@ -9,13 +9,14 @@ exclude =
   python/.eggs/
   python/ray/_private/thirdparty/*
   python/ray/workflow/tests/mock_server.py
-max-line-length = 79
+max-line-length = 88
 inline-quotes = "
 ignore =
   C408
   E121
   E123
   E126
+  E203
   E226
   E24
   E704

.gitpod/Dockerfile

@@ -15,7 +15,7 @@ RUN set -x; apt update \
     && mv bazel.gpg /etc/apt/trusted.gpg.d/ \
     && echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list \
     && apt update && apt install bazel-3.7.2 -y \
-    && pip3 install cython==0.29.26 pytest pandas tree tabulate pexpect sklearn joblib yapf==0.23.0 flake8==3.9.1 mypy==0.782 flake8-quotes flake8-bugbear==21.9.2 setproctitle==1.1.10 psutil \
+    && pip3 install cython==0.29.26 pytest pandas tree tabulate pexpect sklearn joblib black==21.12b0 yapf==0.23.0 flake8==3.9.1 mypy==0.782 flake8-quotes flake8-bugbear==21.9.2 setproctitle==1.1.10 psutil \
     && python3 -c  'print("startup --output_base=/workspace/ray/.bazel-cache\nstartup --host_jvm_args=-Xmx1800m\nbuild --jobs=6")' > /etc/bazel.bazelrc
 
 RUN update-alternatives --install /usr/local/bin/python python /usr/bin/python3 30 \

ci/travis/format.sh

@@ -5,7 +5,10 @@
 # Cause the script to exit if a single command fails
 set -euo pipefail
 
+BLACK_IS_ENABLED=false
+
 FLAKE8_VERSION_REQUIRED="3.9.1"
+BLACK_VERSION_REQUIRED="21.12b0"
 YAPF_VERSION_REQUIRED="0.23.0"
 SHELLCHECK_VERSION_REQUIRED="0.7.1"
 MYPY_VERSION_REQUIRED="0.782"
@@ -13,6 +16,9 @@ MYPY_VERSION_REQUIRED="0.782"
 check_command_exist() {
     VERSION=""
     case "$1" in
+        black)
+            VERSION=$BLACK_VERSION_REQUIRED
+            ;;
         yapf)
             VERSION=$YAPF_VERSION_REQUIRED
             ;;
@@ -35,7 +41,11 @@ check_command_exist() {
     fi
 }
 
-check_command_exist yapf
+if [ "$BLACK_IS_ENABLED" = true ]; then
+  check_command_exist black
+else
+  check_command_exist yapf
+fi
 check_command_exist flake8
 check_command_exist mypy
 
@@ -46,7 +56,11 @@ ROOT="$(git rev-parse --show-toplevel)"
 builtin cd "$ROOT" || exit 1
 
 FLAKE8_VERSION=$(flake8 --version | head -n 1 | awk '{print $1}')
-YAPF_VERSION=$(yapf --version | awk '{print $2}')
+if [ "$BLACK_IS_ENABLED" = true ]; then
+  BLACK_VERSION=$(black --version | awk '{print $2}')
+else
+  YAPF_VERSION=$(yapf --version | awk '{print $2}')
+fi
 SHELLCHECK_VERSION=$(shellcheck --version | awk '/^version:/ {print $2}')
 MYPY_VERSION=$(mypy --version | awk '{print $2}')
 GOOGLE_JAVA_FORMAT_JAR=/tmp/google-java-format-1.7-all-deps.jar
@@ -59,7 +73,11 @@ tool_version_check() {
 }
 
 tool_version_check "flake8" "$FLAKE8_VERSION" "$FLAKE8_VERSION_REQUIRED"
-tool_version_check "yapf" "$YAPF_VERSION" "$YAPF_VERSION_REQUIRED"
+if [ "$BLACK_IS_ENABLED" = true ]; then
+  tool_version_check "black" "$BLACK_VERSION" "$BLACK_VERSION_REQUIRED"
+else
+  tool_version_check "yapf" "$YAPF_VERSION" "$YAPF_VERSION_REQUIRED"
+fi
 tool_version_check "shellcheck" "$SHELLCHECK_VERSION" "$SHELLCHECK_VERSION_REQUIRED"
 tool_version_check "mypy" "$MYPY_VERSION" "$MYPY_VERSION_REQUIRED"
 
@@ -117,6 +135,14 @@ MYPY_FILES=(
     'ray_operator/operator_utils.py'
 )
 
+BLACK_EXCLUDES=(
+    '--extend-exclude' 'python/ray/cloudpickle/*'
+    '--extend-exclude' 'python/build/*'
+    '--extend-exclude' 'python/ray/core/src/ray/gcs/*'
+    '--extend-exclude' 'python/ray/thirdparty_files/*'
+    '--extend-exclude' 'python/ray/_private/thirdparty/*'
+)
+
 YAPF_EXCLUDES=(
     '--exclude' 'python/ray/cloudpickle/*'
     '--exclude' 'python/build/*'
@@ -193,7 +219,11 @@ format_files() {
     done
 
     if [ 0 -lt "${#python_files[@]}" ]; then
-      yapf --in-place "${YAPF_FLAGS[@]}" -- "${python_files[@]}"
+      if [ "$BLACK_IS_ENABLED" = true ]; then
+        black "${python_files[@]}"
+      else
+        yapf --in-place "${YAPF_FLAGS[@]}" -- "${python_files[@]}"
+      fi
     fi
 
     if shellcheck --shell=sh --format=diff - < /dev/null; then
@@ -212,9 +242,15 @@ format_all_scripts() {
     command -v flake8 &> /dev/null;
     HAS_FLAKE8=$?
 
-    echo "$(date)" "YAPF...."
-    git ls-files -- '*.py' "${GIT_LS_EXCLUDES[@]}" | xargs -P 10 \
-      yapf --in-place "${YAPF_EXCLUDES[@]}" "${YAPF_FLAGS[@]}"
+    if [ "$BLACK_IS_ENABLED" = true ]; then
+      echo "$(date)" "Black...."
+      git ls-files -- '*.py' "${GIT_LS_EXCLUDES[@]}" | xargs -P 10 \
+        black "${BLACK_EXCLUDES[@]}"
+    else
+      echo "$(date)" "YAPF...."
+      git ls-files -- '*.py' "${GIT_LS_EXCLUDES[@]}" | xargs -P 10 \
+        yapf --in-place "${YAPF_EXCLUDES[@]}" "${YAPF_FLAGS[@]}"
+    fi
     echo "$(date)" "MYPY...."
     mypy_on_each "${MYPY_FILES[@]}"
     if [ $HAS_FLAKE8 ]; then
@@ -262,16 +298,21 @@ format_all() {
 # for autoformat yet.
 format_changed() {
     # The `if` guard ensures that the list of filenames is not empty, which
-    # could cause yapf to receive 0 positional arguments, making it hang
-    # waiting for STDIN.
+    # could cause the formatter to receive 0 positional arguments, making
+    # Black error and yapf hang waiting for STDIN.
     #
     # `diff-filter=ACRM` and $MERGEBASE is to ensure we only format files that
     # exist on both branches.
     MERGEBASE="$(git merge-base upstream/master HEAD)"
 
     if ! git diff --diff-filter=ACRM --quiet --exit-code "$MERGEBASE" -- '*.py' &>/dev/null; then
-        git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.py' | xargs -P 5 \
-             yapf --in-place "${YAPF_EXCLUDES[@]}" "${YAPF_FLAGS[@]}"
+        if [ "$BLACK_IS_ENABLED" = true ]; then
+            git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.py' | xargs -P 5 \
+                black "${BLACK_EXCLUDES[@]}"
+        else
+            git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.py' | xargs -P 5 \
+                yapf --in-place "${YAPF_EXCLUDES[@]}" "${YAPF_FLAGS[@]}"
+        fi
         if which flake8 >/dev/null; then
             git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.py' | xargs -P 5 \
                  flake8 --config=.flake8

python/ray/tune/requirements-dev.txt

@@ -6,4 +6,5 @@ pandas
 requests
 tabulate
 tensorflow
+black==21.12b0
 yapf==0.23.0

python/requirements_linters.txt

@@ -3,4 +3,5 @@ flake8-comprehensions
 flake8-quotes==2.0.0
 flake8-bugbear==21.9.2
 mypy==0.782
+black==21.12b0
 yapf==0.23.0