diff --git a/build-docker.sh b/build-docker.sh index 5e6b4a2ea..4e3a44d93 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -52,5 +52,5 @@ if [ ! $SKIP_EXAMPLES ]; then fi if [ $OUTPUT_SHA ]; then - echo "$IMAGE_SHA" | sed 's/sha256://' + echo "${IMAGE_SHA/sha256:/}" fi diff --git a/ci/asan_tests/ray-project/project.yaml b/ci/asan_tests/ray-project/project.yaml index 22add2a22..a41435213 100644 --- a/ci/asan_tests/ray-project/project.yaml +++ b/ci/asan_tests/ray-project/project.yaml @@ -20,8 +20,7 @@ commands: - name: Run command: ./run_asan_tests.sh run - name: Recompile - command: | - git_sha={{git_sha}} ./run_asan_tests.sh recompile + command: ./run_asan_tests.sh recompile {{git_sha}} params: - name: git_sha # Ray version string. default: "master" diff --git a/ci/asan_tests/run_asan_tests.sh b/ci/asan_tests/run_asan_tests.sh index ff67c442a..8cb8a25a8 100755 --- a/ci/asan_tests/run_asan_tests.sh +++ b/ci/asan_tests/run_asan_tests.sh @@ -47,7 +47,7 @@ asan_run() { asan_recompile() { git fetch - git checkout "${git_sha}" + git checkout "$1" asan_install || true } diff --git a/ci/jenkins_tests/entry_point.sh b/ci/jenkins_tests/entry_point.sh index 78c09b3c2..ec701cdff 100755 --- a/ci/jenkins_tests/entry_point.sh +++ b/ci/jenkins_tests/entry_point.sh @@ -12,33 +12,32 @@ SHM_SIZE="4G" ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -SUPPRESS_OUTPUT=$ROOT_DIR/../suppress_output - function retry { local n=1 local max=3 while true; do - "$@" && break || { - if [[ $n -lt $max ]]; then - ((n++)) - echo "Command failed. Attempt $n/$max:" - else - echo "The command has failed after $n attempts." - exit 1 - fi - } + if "$@"; then + break + fi + if [ $n -lt $max ]; then + ((n++)) + echo "Command failed. Attempt $n/$max:" + else + echo "The command has failed after $n attempts." + exit 1 + fi done } -if [[ ! -z "$RUN_TUNE_TESTS" ]]; then +if [[ -n "$RUN_TUNE_TESTS" ]]; then retry bash "$ROOT_DIR"/run_tune_tests.sh ${MEMORY_SIZE} ${SHM_SIZE} fi -if [[ ! -z "$RUN_DOC_TESTS" ]]; then +if [[ -n "$RUN_DOC_TESTS" ]]; then retry bash "$ROOT_DIR"/run_doc_tests.sh ${MEMORY_SIZE} ${SHM_SIZE} fi -if [[ ! -z "$RUN_SGD_TESTS" ]]; then +if [[ -n "$RUN_SGD_TESTS" ]]; then retry bash "$ROOT_DIR"/run_sgd_tests.sh ${MEMORY_SIZE} ${SHM_SIZE} fi \ No newline at end of file diff --git a/ci/jenkins_tests/run_multi_node_tests.sh b/ci/jenkins_tests/run_multi_node_tests.sh index f04b0e645..736a9ca12 100755 --- a/ci/jenkins_tests/run_multi_node_tests.sh +++ b/ci/jenkins_tests/run_multi_node_tests.sh @@ -2,7 +2,7 @@ # This file exists for Jenkins compatiblity -ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")" && pwd)" export RUN_TUNE_TESTS=1 export RUN_DOC_TESTS=1 diff --git a/ci/jenkins_tests/run_tune_tests.sh b/ci/jenkins_tests/run_tune_tests.sh index 7998c0c0e..795ff59ce 100755 --- a/ci/jenkins_tests/run_tune_tests.sh +++ b/ci/jenkins_tests/run_tune_tests.sh @@ -99,7 +99,7 @@ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/doc/source/tune/_tutorials/tune-sklearn.py -# if [[ ! -z "$SIGOPT_KEY" ]]; then +# if [ -n "$SIGOPT_KEY" ]; then # $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 -e SIGOPT_KEY "$DOCKER_SHA" \ # python /ray/python/ray/tune/examples/sigopt_example.py \ # --smoke-test diff --git a/ci/keep_alive b/ci/keep_alive index 4489ea904..6e129a8bb 100755 --- a/ci/keep_alive +++ b/ci/keep_alive @@ -13,7 +13,7 @@ watchdog() { kill -SIGKILL $PID } -watchdog & 2>/dev/null +watchdog 2>/dev/null & WATCHDOG_PID=$! time "$@" diff --git a/ci/suppress_output b/ci/suppress_output index 15b0825b9..19deba61d 100755 --- a/ci/suppress_output +++ b/ci/suppress_output @@ -16,7 +16,7 @@ watchdog() { kill -SIGKILL $PID } -watchdog & 2>/dev/null +watchdog 2>/dev/null & WATCHDOG_PID=$! time "$@" >"$TMPFILE" 2>&1 diff --git a/ci/travis/bazel-format.sh b/ci/travis/bazel-format.sh index 1ebcb68b7..86bbb9f8c 100755 --- a/ci/travis/bazel-format.sh +++ b/ci/travis/bazel-format.sh @@ -20,7 +20,7 @@ function usage() RUN_TYPE="diff" # Parse options -while [[ $# > 0 ]]; do +while [ $# -gt 0 ]; do key="$1" case $key in -h|--help) @@ -37,7 +37,7 @@ while [[ $# > 0 ]]; do echo "ERROR: unknown option \"$key\"" echo usage - exit -1 + exit 1 ;; esac shift diff --git a/ci/travis/build-autoscaler-images.sh b/ci/travis/build-autoscaler-images.sh index 146b14d9e..c41bf28fe 100755 --- a/ci/travis/build-autoscaler-images.sh +++ b/ci/travis/build-autoscaler-images.sh @@ -13,7 +13,7 @@ DOCKER_USERNAME="raytravisbot" if [[ "$TRAVIS" == "true" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - wheel=$(cd $ROOT_DIR/.whl; ls | grep cp36m-manylinux) + wheel="$(basename "$ROOT_DIR"/.whl/*cp36m-manylinux*)" commit_sha=$(echo "$TRAVIS_COMMIT" | head -c 6) cp -r "$ROOT_DIR"/.whl "$ROOT_DIR"/docker/autoscaler/.whl @@ -21,7 +21,7 @@ if [[ "$TRAVIS" == "true" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then docker build \ --build-arg WHEEL_PATH=".whl/$wheel" \ - --build-arg WHEEL_NAME=$wheel \ + --build-arg WHEEL_NAME="$wheel" \ -t rayproject/autoscaler:"$commit_sha" \ "$ROOT_DIR"/docker/autoscaler diff --git a/ci/travis/format.sh b/ci/travis/format.sh index bdb44481d..be927c068 100755 --- a/ci/travis/format.sh +++ b/ci/travis/format.sh @@ -57,8 +57,8 @@ tool_version_check() { fi } -tool_version_check "flake8" $FLAKE8_VERSION $FLAKE8_VERSION_REQUIRED -tool_version_check "yapf" $YAPF_VERSION $YAPF_VERSION_REQUIRED +tool_version_check "flake8" "$FLAKE8_VERSION" "$FLAKE8_VERSION_REQUIRED" +tool_version_check "yapf" "$YAPF_VERSION" "$YAPF_VERSION_REQUIRED" if which clang-format >/dev/null; then CLANG_FORMAT_VERSION=$(clang-format --version | awk '{print $3}') diff --git a/ci/travis/install-bazel.sh b/ci/travis/install-bazel.sh index 1a8ab6ef8..4f64a677b 100755 --- a/ci/travis/install-bazel.sh +++ b/ci/travis/install-bazel.sh @@ -32,7 +32,7 @@ esac { git ls-files -s 2>/dev/null || true; } | ( set +x missing_symlinks=() - while read -r mode digest sn path; do + while read -r mode _ _ path; do if [ "${mode}" = 120000 ]; then test -L "${path}" || missing_symlinks+=("${paths}") fi diff --git a/ci/travis/install-dependencies.sh b/ci/travis/install-dependencies.sh index 96a78df25..1301c7a8a 100755 --- a/ci/travis/install-dependencies.sh +++ b/ci/travis/install-dependencies.sh @@ -250,8 +250,7 @@ install_dependencies() { # after n seconds. local status="0"; local errmsg=""; - for i in {1..3}; - do + for _ in {1..3}; do errmsg=$(CC=gcc pip install "${pip_packages[@]}" 2>&1) && break; status=$errmsg && echo "'pip install ...' failed, will retry after n seconds!" && sleep 30; done diff --git a/ci/travis/test-wheels.sh b/ci/travis/test-wheels.sh index 60cca850b..24fc848d7 100755 --- a/ci/travis/test-wheels.sh +++ b/ci/travis/test-wheels.sh @@ -29,15 +29,16 @@ function retry { local max=3 while true; do - "$@" && break || { - if [[ $n -lt $max ]]; then - ((n++)) - echo "Command failed. Attempt $n/$max:" - else - echo "The command has failed after $n attempts." - exit 1 - fi - } + if "$@"; then + break + fi + if [ $n -lt $max ]; then + ((n++)) + echo "Command failed. Attempt $n/$max:" + else + echo "The command has failed after $n attempts." + exit 1 + fi done } @@ -61,7 +62,7 @@ if [[ "$platform" == "linux" ]]; then PIP_CMD="$HOME/miniconda3/bin/pip" # Find the right wheel by grepping for the Python version. - PYTHON_WHEEL=$(find "$ROOT_DIR/../../.whl" -type f -maxdepth 1 -print | grep -m1 "$PY_WHEEL_VERSION") + PYTHON_WHEEL="$(printf "%s\n" "$ROOT_DIR"/../../.whl/*"$PY_WHEEL_VERSION"* | head -n 1)" # Install the wheel. "$PIP_CMD" install -q "$PYTHON_WHEEL" @@ -73,8 +74,6 @@ if [[ "$platform" == "linux" ]]; then "$PIP_CMD" install -q aiohttp google grpcio pytest requests # Run a simple test script to make sure that the wheel works. - INSTALLED_RAY_DIRECTORY=$(dirname "$($PYTHON_EXE -u -c "import ray; print(ray.__file__)" | tail -n1)") - for SCRIPT in "${TEST_SCRIPTS[@]}"; do retry "$PYTHON_EXE" "$SCRIPT" done @@ -84,7 +83,7 @@ if [[ "$platform" == "linux" ]]; then done # Check that the other wheels are present. - NUMBER_OF_WHEELS=$(ls -1q "$ROOT_DIR"/../../.whl/*.whl | wc -l) + NUMBER_OF_WHEELS="$(find "$ROOT_DIR"/../../.whl/ -mindepth 1 -maxdepth 1 -name "*.whl" | wc -l)" if [[ "$NUMBER_OF_WHEELS" != "3" ]]; then echo "Wrong number of wheels found." ls -l "$ROOT_DIR/../.whl/" @@ -107,7 +106,7 @@ elif [[ "$platform" == "macosx" ]]; then PIP_CMD="$(dirname "$PYTHON_EXE")/pip$PY_MM" # Find the appropriate wheel by grepping for the Python version. - PYTHON_WHEEL=$(find "$ROOT_DIR/../../.whl" -type f -maxdepth 1 -print | grep -m1 "$PY_WHEEL_VERSION") + PYTHON_WHEEL="$(printf "%s\n" "$ROOT_DIR"/../../.whl/*"$PY_WHEEL_VERSION"* | head -n 1)" # Install the wheel. "$PIP_CMD" install -q "$PYTHON_WHEEL" @@ -116,7 +115,6 @@ elif [[ "$platform" == "macosx" ]]; then "$PIP_CMD" install -q aiohttp google grpcio pytest requests # Run a simple test script to make sure that the wheel works. - INSTALLED_RAY_DIRECTORY=$(dirname "$($PYTHON_EXE -u -c "import ray; print(ray.__file__)" | tail -n1)") for SCRIPT in "${TEST_SCRIPTS[@]}"; do retry "$PYTHON_EXE" "$SCRIPT" done diff --git a/doc/dev/bin/pip_download_test.sh b/doc/dev/bin/pip_download_test.sh index 0690b4164..c0373e021 100755 --- a/doc/dev/bin/pip_download_test.sh +++ b/doc/dev/bin/pip_download_test.sh @@ -32,11 +32,13 @@ do env_name="${RAY_VERSION}-${PYTHON_VERSION}-env" conda create -y -n "${env_name}" python=${PYTHON_VERSION} conda activate "${env_name}" - echo "\n\n\n=========================================================" + printf "\n\n\n" + echo "=========================================================" echo "Python version." python --version echo "This should be equal to ${PYTHON_VERSION}" - echo "=========================================================\n\n\n" + echo "=========================================================" + printf "\n\n\n" pip install redis==3.3.2 pip install msgpack==0.6.2 @@ -45,13 +47,15 @@ do pip install --index-url https://test.pypi.org/simple/ ray failed=false - echo "\n\n\n=========================================================" + printf "\n\n\n" + echo "=========================================================" if python sanity_check.py; then echo "PYTHON ${PYTHON_VERSION} succeed sanity check." else failed=true fi - echo "=========================================================\n\n\n" + echo "=========================================================" + printf "\n\n\n" conda deactivate conda remove -y --name "${env_name}" --all diff --git a/doc/examples/lm/preprocess.sh b/doc/examples/lm/preprocess.sh index ceb6fd414..46efce24c 100644 --- a/doc/examples/lm/preprocess.sh +++ b/doc/examples/lm/preprocess.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -euo pipefail + cd ~/efs/lm # download the dataset diff --git a/doc/examples/lm/ray_train.sh b/doc/examples/lm/ray_train.sh index 8fe014007..c0c406f75 100644 --- a/doc/examples/lm/ray_train.sh +++ b/doc/examples/lm/ray_train.sh @@ -4,7 +4,7 @@ TOTAL_UPDATES=125000 # Total number of training steps WARMUP_UPDATES=10000 # Warmup the learning rate over this many updates PEAK_LR=0.0005 # Peak learning rate, adjust as needed TOKENS_PER_SAMPLE=512 # Max sequence length -MAX_POSITIONS=512 # Num. positional embeddings (usually same as above) +#MAX_POSITIONS=512 # Num. positional embeddings (usually same as above) MAX_SENTENCES=8 # Number of sequences per batch on one GPU (batch size) FIX_BATCH_SIZE=2048 # Number of batch size in total (max_sentences * update_freq * n_gpus) SAVE_INTERVAL_UPDATES=1000 # save a checkpoint every N updates diff --git a/doc/examples/plot_example-lm.rst b/doc/examples/plot_example-lm.rst index 4cee6826d..843a7e782 100644 --- a/doc/examples/plot_example-lm.rst +++ b/doc/examples/plot_example-lm.rst @@ -266,7 +266,7 @@ To start training, run `following commands