Shellcheck rewrites (#9597)

* Fix SC2001: See if you can use ${variable//search/replace} instead.

* Fix SC2010: Don't use ls | grep. Use a glob or a for loop with a condition to allow non-alphanumeric filenames.

* Fix SC2012: Use find instead of ls to better handle non-alphanumeric filenames.

* Fix SC2015: Note that A && B || C is not if-then-else. C may run when A is true.

* Fix SC2028: echo may not expand escape sequences. Use printf.

* Fix SC2034: variable appears unused. Verify use (or export if used externally).

* Fix SC2035: Use ./*glob* or -- *glob* so names with dashes won't become options.

* Fix SC2071: > is for string comparisons. Use -gt instead.

* Fix SC2154: variable is referenced but not assigned

* Fix SC2164: Use 'cd ... || exit' or 'cd ... || return' in case cd fails.

* Fix SC2188: This redirection doesn't have a command. Move to its command (or use 'true' as no-op).

* Fix SC2236: Use -n instead of ! -z.

* Fix SC2242: Can only exit with status 0-255. Other data should be written to stdout/stderr.

* Fix SC2086: Double quote to prevent globbing and word splitting.

Co-authored-by: Mehrdad <noreply@github.com>

build-docker.sh

@@ -52,5 +52,5 @@ if [ ! $SKIP_EXAMPLES ]; then
 fi
 
 if [ $OUTPUT_SHA ]; then
-    echo "$IMAGE_SHA" | sed 's/sha256://'
+    echo "${IMAGE_SHA/sha256:/}"
 fi

ci/asan_tests/ray-project/project.yaml

@@ -20,8 +20,7 @@ commands:
   - name: Run
     command: ./run_asan_tests.sh run
   - name: Recompile
-    command: |
-      git_sha={{git_sha}} ./run_asan_tests.sh recompile
+    command: ./run_asan_tests.sh recompile {{git_sha}}
     params:
     - name: git_sha # Ray version string.
       default: "master"

ci/asan_tests/run_asan_tests.sh

@@ -47,7 +47,7 @@ asan_run() {
 
 asan_recompile() {
   git fetch
-  git checkout "${git_sha}"
+  git checkout "$1"
   asan_install || true
 }
 

ci/jenkins_tests/entry_point.sh

@@ -12,33 +12,32 @@ SHM_SIZE="4G"
 
 ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 
-SUPPRESS_OUTPUT=$ROOT_DIR/../suppress_output
-
 function retry {
   local n=1
   local max=3
 
   while true; do
-    "$@" && break || {
-      if [[ $n -lt $max ]]; then
-        ((n++))
-        echo "Command failed. Attempt $n/$max:"
-      else
-        echo "The command has failed after $n attempts."
-        exit 1
-      fi
-    }
+    if "$@"; then
+      break
+    fi
+    if [ $n -lt $max ]; then
+      ((n++))
+      echo "Command failed. Attempt $n/$max:"
+    else
+      echo "The command has failed after $n attempts."
+      exit 1
+    fi
   done
 }
 
-if [[ ! -z "$RUN_TUNE_TESTS" ]]; then 
+if [[ -n "$RUN_TUNE_TESTS" ]]; then 
     retry bash "$ROOT_DIR"/run_tune_tests.sh ${MEMORY_SIZE} ${SHM_SIZE}
 fi
 
-if [[ ! -z "$RUN_DOC_TESTS" ]]; then
+if [[ -n "$RUN_DOC_TESTS" ]]; then
     retry bash "$ROOT_DIR"/run_doc_tests.sh ${MEMORY_SIZE} ${SHM_SIZE}
 fi
 
-if [[ ! -z "$RUN_SGD_TESTS" ]]; then
+if [[ -n "$RUN_SGD_TESTS" ]]; then
     retry bash "$ROOT_DIR"/run_sgd_tests.sh ${MEMORY_SIZE} ${SHM_SIZE}
 fi

ci/jenkins_tests/run_multi_node_tests.sh

@@ -2,7 +2,7 @@
 
 # This file exists for Jenkins compatiblity
 
-ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")" && pwd)"
 
 export RUN_TUNE_TESTS=1
 export RUN_DOC_TESTS=1

ci/jenkins_tests/run_tune_tests.sh

@@ -99,7 +99,7 @@ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE}
 $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
     python /ray/doc/source/tune/_tutorials/tune-sklearn.py
 
-# if [[ ! -z "$SIGOPT_KEY" ]]; then
+# if [ -n "$SIGOPT_KEY" ]; then
 #     $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 -e SIGOPT_KEY "$DOCKER_SHA" \
 #         python /ray/python/ray/tune/examples/sigopt_example.py \
 #         --smoke-test

ci/keep_alive

@@ -13,7 +13,7 @@ watchdog() {
     kill -SIGKILL $PID
 }
 
-watchdog & 2>/dev/null
+watchdog 2>/dev/null &
 WATCHDOG_PID=$!
 
 time "$@"

ci/suppress_output

@@ -16,7 +16,7 @@ watchdog() {
     kill -SIGKILL $PID
 }
 
-watchdog & 2>/dev/null
+watchdog 2>/dev/null &
 WATCHDOG_PID=$!
 
 time "$@" >"$TMPFILE" 2>&1

ci/travis/bazel-format.sh

@@ -20,7 +20,7 @@ function usage()
 RUN_TYPE="diff"
 
 # Parse options
-while [[ $# > 0 ]]; do
+while [ $# -gt 0 ]; do
   key="$1"
   case $key in
     -h|--help)
@@ -37,7 +37,7 @@ while [[ $# > 0 ]]; do
       echo "ERROR: unknown option \"$key\""
       echo
       usage
-      exit -1
+      exit 1
       ;;
   esac
   shift

ci/travis/build-autoscaler-images.sh

@@ -13,7 +13,7 @@ DOCKER_USERNAME="raytravisbot"
 if [[ "$TRAVIS" == "true" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then
     echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
 
-    wheel=$(cd $ROOT_DIR/.whl; ls | grep cp36m-manylinux)
+    wheel="$(basename "$ROOT_DIR"/.whl/*cp36m-manylinux*)"
     commit_sha=$(echo "$TRAVIS_COMMIT" | head -c 6)
     cp -r "$ROOT_DIR"/.whl "$ROOT_DIR"/docker/autoscaler/.whl
 
@@ -21,7 +21,7 @@ if [[ "$TRAVIS" == "true" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then
 
     docker build \
         --build-arg WHEEL_PATH=".whl/$wheel" \
-        --build-arg WHEEL_NAME=$wheel \
+        --build-arg WHEEL_NAME="$wheel" \
         -t rayproject/autoscaler:"$commit_sha" \
         "$ROOT_DIR"/docker/autoscaler
 

ci/travis/format.sh

@@ -57,8 +57,8 @@ tool_version_check() {
     fi
 }
 
-tool_version_check "flake8" $FLAKE8_VERSION $FLAKE8_VERSION_REQUIRED
-tool_version_check "yapf" $YAPF_VERSION $YAPF_VERSION_REQUIRED
+tool_version_check "flake8" "$FLAKE8_VERSION" "$FLAKE8_VERSION_REQUIRED"
+tool_version_check "yapf" "$YAPF_VERSION" "$YAPF_VERSION_REQUIRED"
 
 if which clang-format >/dev/null; then
   CLANG_FORMAT_VERSION=$(clang-format --version | awk '{print $3}')

ci/travis/install-bazel.sh

@@ -32,7 +32,7 @@ esac
 { git ls-files -s 2>/dev/null || true; } | (
   set +x
   missing_symlinks=()
-  while read -r mode digest sn path; do
+  while read -r mode _ _ path; do
     if [ "${mode}" = 120000 ]; then
       test -L "${path}" || missing_symlinks+=("${paths}")
     fi

ci/travis/install-dependencies.sh

@@ -250,8 +250,7 @@ install_dependencies() {
     # after n seconds.
     local status="0";
     local errmsg="";
-    for i in {1..3};
-    do
+    for _ in {1..3}; do
       errmsg=$(CC=gcc pip install "${pip_packages[@]}" 2>&1) && break;
       status=$errmsg && echo "'pip install ...' failed, will retry after n seconds!" && sleep 30;
     done

ci/travis/test-wheels.sh

@@ -29,15 +29,16 @@ function retry {
   local max=3
 
   while true; do
-    "$@" && break || {
-      if [[ $n -lt $max ]]; then
-        ((n++))
-        echo "Command failed. Attempt $n/$max:"
-      else
-        echo "The command has failed after $n attempts."
-        exit 1
-      fi
-    }
+    if "$@"; then
+      break
+    fi
+    if [ $n -lt $max ]; then
+      ((n++))
+      echo "Command failed. Attempt $n/$max:"
+    else
+      echo "The command has failed after $n attempts."
+      exit 1
+    fi
   done
 }
 
@@ -61,7 +62,7 @@ if [[ "$platform" == "linux" ]]; then
     PIP_CMD="$HOME/miniconda3/bin/pip"
 
     # Find the right wheel by grepping for the Python version.
-    PYTHON_WHEEL=$(find "$ROOT_DIR/../../.whl" -type f -maxdepth 1 -print | grep -m1 "$PY_WHEEL_VERSION")
+    PYTHON_WHEEL="$(printf "%s\n" "$ROOT_DIR"/../../.whl/*"$PY_WHEEL_VERSION"* | head -n 1)"
 
     # Install the wheel.
     "$PIP_CMD" install -q "$PYTHON_WHEEL"
@@ -73,8 +74,6 @@ if [[ "$platform" == "linux" ]]; then
     "$PIP_CMD" install -q aiohttp google grpcio pytest requests
 
     # Run a simple test script to make sure that the wheel works.
-    INSTALLED_RAY_DIRECTORY=$(dirname "$($PYTHON_EXE -u -c "import ray; print(ray.__file__)" | tail -n1)")
-
     for SCRIPT in "${TEST_SCRIPTS[@]}"; do
         retry "$PYTHON_EXE" "$SCRIPT"
     done
@@ -84,7 +83,7 @@ if [[ "$platform" == "linux" ]]; then
   done
 
   # Check that the other wheels are present.
-  NUMBER_OF_WHEELS=$(ls -1q "$ROOT_DIR"/../../.whl/*.whl | wc -l)
+  NUMBER_OF_WHEELS="$(find "$ROOT_DIR"/../../.whl/ -mindepth 1 -maxdepth 1 -name "*.whl" | wc -l)"
   if [[ "$NUMBER_OF_WHEELS" != "3" ]]; then
     echo "Wrong number of wheels found."
     ls -l "$ROOT_DIR/../.whl/"
@@ -107,7 +106,7 @@ elif [[ "$platform" == "macosx" ]]; then
     PIP_CMD="$(dirname "$PYTHON_EXE")/pip$PY_MM"
 
     # Find the appropriate wheel by grepping for the Python version.
-    PYTHON_WHEEL=$(find "$ROOT_DIR/../../.whl" -type f -maxdepth 1 -print | grep -m1 "$PY_WHEEL_VERSION")
+    PYTHON_WHEEL="$(printf "%s\n" "$ROOT_DIR"/../../.whl/*"$PY_WHEEL_VERSION"* | head -n 1)"
 
     # Install the wheel.
     "$PIP_CMD" install -q "$PYTHON_WHEEL"
@@ -116,7 +115,6 @@ elif [[ "$platform" == "macosx" ]]; then
     "$PIP_CMD" install -q aiohttp google grpcio pytest requests
 
     # Run a simple test script to make sure that the wheel works.
-    INSTALLED_RAY_DIRECTORY=$(dirname "$($PYTHON_EXE -u -c "import ray; print(ray.__file__)" | tail -n1)")
     for SCRIPT in "${TEST_SCRIPTS[@]}"; do
       retry "$PYTHON_EXE" "$SCRIPT"
     done

doc/dev/bin/pip_download_test.sh

@@ -32,11 +32,13 @@ do
     env_name="${RAY_VERSION}-${PYTHON_VERSION}-env"
     conda create -y -n "${env_name}" python=${PYTHON_VERSION}
     conda activate "${env_name}"
-    echo "\n\n\n========================================================="
+    printf "\n\n\n"
+    echo "========================================================="
     echo "Python version."
     python --version
     echo "This should be equal to ${PYTHON_VERSION}"
-    echo "=========================================================\n\n\n"
+    echo "========================================================="
+    printf "\n\n\n"
 
     pip install redis==3.3.2
     pip install msgpack==0.6.2
@@ -45,13 +47,15 @@ do
     pip install --index-url https://test.pypi.org/simple/ ray
 
     failed=false
-    echo "\n\n\n========================================================="
+    printf "\n\n\n"
+    echo "========================================================="
     if python sanity_check.py; then
         echo "PYTHON ${PYTHON_VERSION} succeed sanity check."
     else
         failed=true
     fi
-    echo "=========================================================\n\n\n"
+    echo "========================================================="
+    printf "\n\n\n"
 
     conda deactivate
     conda remove -y --name "${env_name}" --all

doc/examples/lm/preprocess.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -euo pipefail
+
 cd ~/efs/lm
 
 # download the dataset

doc/examples/lm/ray_train.sh

@@ -4,7 +4,7 @@ TOTAL_UPDATES=125000       # Total number of training steps
 WARMUP_UPDATES=10000       # Warmup the learning rate over this many updates
 PEAK_LR=0.0005             # Peak learning rate, adjust as needed
 TOKENS_PER_SAMPLE=512      # Max sequence length
-MAX_POSITIONS=512          # Num. positional embeddings (usually same as above)
+#MAX_POSITIONS=512         # Num. positional embeddings (usually same as above)
 MAX_SENTENCES=8            # Number of sequences per batch on one GPU (batch size)
 FIX_BATCH_SIZE=2048        # Number of batch size in total (max_sentences * update_freq * n_gpus)
 SAVE_INTERVAL_UPDATES=1000 # save a checkpoint every N updates

doc/examples/plot_example-lm.rst

@@ -266,7 +266,7 @@ To start training, run `following commands <https://github.com/ray-project/ray/t
   WARMUP_UPDATES=10000       # Warmup the learning rate over this many updates
   PEAK_LR=0.0005             # Peak learning rate, adjust as needed
   TOKENS_PER_SAMPLE=512      # Max sequence length
-  MAX_POSITIONS=512          # Num. positional embeddings (usually same as above)
+  #MAX_POSITIONS=512         # Num. positional embeddings (usually same as above)
   MAX_SENTENCES=8            # Number of sequences per batch on one GPU (batch size)
   FIX_BATCH_SIZE=2048        # Number of batch size in total (max_sentences * update_freq * n_gpus)
   SAVE_INTERVAL_UPDATES=1000 # save a checkpoint every N updates

doc/tools/install-prometheus-server.sh

@@ -17,7 +17,7 @@ elif [[ "$unamestr" == "Darwin" ]]; then
     PACKAGE_NAME=prometheus-2.8.0.darwin-amd64
 else
     echo "Downloading abort: Unrecognized platform."
-    exit -1
+    exit 1
 fi
 
 URL=https://github.com/prometheus/prometheus/releases/download/v2.8.0/$PACKAGE_NAME.tar.gz

python/build-wheel-manylinux1.sh

@@ -72,6 +72,8 @@ done
 
 # Rename the wheels so that they can be uploaded to PyPI. TODO(rkn): This is a
 # hack, we should use auditwheel instead.
-pushd .whl
-  find *.whl -exec bash -c 'mv $1 ${1//linux/manylinux1}' bash {} \;
-popd
+for path in .whl/*.whl; do
+  if [ -f "${path}" ]; then
+    mv "${path}" "${path//linux/manylinux1}"
+  fi
+done