[Tune] Pin Tune Dependencies (#13027)

Co-authored-by: Ian <ian.rodney@gmail.com>

.travis.yml

@@ -358,9 +358,6 @@ matrix:
       env:
         - TUNE_TESTING=1
         - PYTHON=3.6
-        - TF_VERSION=2.2.0
-        - TFP_VERSION=0.8
-        - TORCH_VERSION=1.7
         - PYTHONWARNINGS=ignore
       install:
         - . ./ci/travis/ci.sh init RAY_CI_TUNE_AFFECTED
@@ -378,9 +375,6 @@ matrix:
       env:
         - SGD_TESTING=1
         - PYTHON=3.6
-        - TF_VERSION=2.1.0
-        - TFP_VERSION=0.8
-        - TORCH_VERSION=1.7
         - PYTHONWARNINGS=ignore
       install:
         - . ./ci/travis/ci.sh init RAY_CI_SGD_AFFECTED
@@ -398,9 +392,6 @@ matrix:
       env:
         - DOC_TESTING=1
         - PYTHON=3.6
-        - TF_VERSION=2.1.0
-        - TFP_VERSION=0.8
-        - TORCH_VERSION=1.7
         - PYTHONWARNINGS=ignore
       install:
         - . ./ci/travis/ci.sh init RAY_CI_PYTHON_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_DOC_AFFECTED
@@ -416,9 +407,6 @@ matrix:
       env:
         - PYTHON=3.7 TUNE_TESTING=1
         - INSTALL_HOROVOD=1
-        - TF_VERSION=2.1.0
-        - TFP_VERSION=0.8
-        - TORCH_VERSION=1.7
         - PYTHONWARNINGS=ignore
       install:
         - . ./ci/travis/ci.sh init RAY_CI_TUNE_AFFECTED,RAY_CI_SGD_AFFECTED

ci/travis/build-docker-images.py

@@ -179,6 +179,8 @@ def build_ray_ml():
     root_dir = _get_root_dir()
     requirement_files = glob.glob(
         f"{_get_root_dir()}/python/requirements*.txt")
+    requirement_files.extend(
+        glob.glob(f"{_get_root_dir()}/python/requirements/*.txt"))
     for fl in requirement_files:
         shutil.copy(fl, os.path.join(root_dir, "docker/ray-ml/"))
     ray_ml_images = _build_cpu_gpu_images("ray-ml")

ci/travis/install-dependencies.sh

@@ -249,6 +249,8 @@ install_dependencies() {
   pip install --no-clean dm-tree  # --no-clean is due to: https://github.com/deepmind/tree/issues/5
 
   if [ -n "${PYTHON-}" ]; then
+    # Remove this entire section once RLlib and Serve dependencies are fixed.
+    if [ "${DOC_TESTING-}" != 1 ] && [ "${SGD_TESTING-}" != 1 ] && [ "${TUNE_TESTING-}" != 1 ]; then
       # PyTorch is installed first since we are using a "-f" directive to find the wheels.
       # We want to install the CPU version only.
       local torch_url="https://download.pytorch.org/whl/torch_stable.html"
@@ -256,6 +258,7 @@ install_dependencies() {
         darwin*) pip install torch torchvision;;
         *) pip install torch==1.7.0+cpu torchvision==0.8.1+cpu -f "${torch_url}";;
       esac
+    fi
 
     # Try n times; we often encounter OpenSSL.SSL.WantReadError (or others)
     # that break the entire CI job: Simply retry installation in this case
@@ -294,21 +297,22 @@ install_dependencies() {
 
   # Additional Tune test dependencies.
   if [ "${TUNE_TESTING-}" = 1 ]; then
-    pip install -r "${WORKSPACE_DIR}"/python/requirements_tune.txt
+    pip install -r "${WORKSPACE_DIR}"/python/requirements/requirements_tune.txt
   fi
 
   # Additional RaySGD test dependencies.
   if [ "${SGD_TESTING-}" = 1 ]; then
-    pip install -r "${WORKSPACE_DIR}"/python/requirements_tune.txt
+    pip install -r "${WORKSPACE_DIR}"/python/requirements/requirements_tune.txt
     # TODO: eventually have a separate requirements file for Ray SGD.
   fi
 
   # Additional Doc test dependencies.
   if [ "${DOC_TESTING-}" = 1 ]; then
-    pip install -r "${WORKSPACE_DIR}"/python/requirements_tune.txt
+    pip install -r "${WORKSPACE_DIR}"/python/requirements/requirements_tune.txt
   fi
 
-
+  # Remove this entire section once RLlib and Serve dependencies are fixed.
+  if [ "${DOC_TESTING-}" != 1 ] && [ "${SGD_TESTING-}" != 1 ] && [ "${TUNE_TESTING-}" != 1 ]; then
     # If CI has deemed that a different version of Tensorflow or Torch
     # should be installed, then upgrade/downgrade to that specific version.
     if [ -n "${TORCH_VERSION-}" ] || [ -n "${TFP_VERSION-}" ] || [ -n "${TF_VERSION-}" ]; then
@@ -321,6 +325,7 @@ install_dependencies() {
         torch=="${TORCH_VERSION-1.7}" torchvision=="${TORCHVISION_VERSION}" \
         tensorflow=="${TF_VERSION-2.2.0}" gym
     fi
+  fi
 
   # Additional Tune dependency for Horovod.
   # This must be run last (i.e., torch cannot be re-installed after this)

docker/ray-ml/Dockerfile

@@ -5,7 +5,7 @@ FROM rayproject/ray:nightly"$GPU"
 COPY requirements.txt ./
 COPY requirements_ml_docker.txt ./
 COPY requirements_rllib.txt ./
-COPY requirements_tune.txt ./
+COPY requirements_tune.txt ./requirements_tune.txt
 
 RUN sudo apt-get update \
     && sudo apt-get install -y gcc \
@@ -18,5 +18,6 @@ RUN sudo apt-get update \
     # Remove dataclasses & typing because they are included in Py3.7
     && $HOME/anaconda3/bin/pip uninstall dataclasses typing -y  \
     && sudo rm requirements.txt && sudo rm requirements_ml_docker.txt \
+    && sudo rm requirements_tune.txt && sudo rm requirements_rllib.txt \
     && sudo apt-get clean
 

python/requirements/requirements_tune.txt

@@ -0,0 +1,47 @@
+ax-platform==0.1.9; python_version < '3.7'
+ax-platform==0.1.19; python_version >= '3.7'
+bayesian-optimization==1.2.0
+ConfigSpace==0.4.10
+dragonfly-opt==0.1.6
+gluoncv==0.9.0
+gpy==1.9.9
+gym[atari]==0.18.0
+h5py==3.1.0
+hpbandster==0.7.4
+hyperopt==0.2.5
+jupyter==1.0.0
+keras==2.4.3
+kubernetes==12.0.1
+lightgbm==3.1.1
+matplotlib==3.3.3
+mlflow==1.13.0
+mxnet==1.7.0.post1
+nevergrad==0.4.2.post5
+optuna==2.3.0
+pytest-remotedata==0.3.2
+pytorch-lightning-bolts==0.2.5
+pytorch-lightning==1.0.3
+scikit-learn==0.22.2
+scikit-optimize==0.8.1
+sigopt==5.7.0
+smart_open==4.0.1
+tensorflow-probability==0.11.1
+timm==0.3.2
+
+
+torch==1.7.0;sys_platform=="darwin"
+torchvision==0.8.1;sys_platform=="darwin"
+
+# On non-OSX machines only install CPU version of torch and torchvision
+-f https://download.pytorch.org/whl/torch_stable.html
+torch==1.7.0+cpu;sys_platform!="darwin"
+-f https://download.pytorch.org/whl/torch_stable.html
+torchvision==0.8.1+cpu;sys_platform!="darwin"
+
+
+transformers==3.1
+tune-sklearn==0.2.1
+wandb==0.10.12
+xgboost==1.3.0.post0
+xgboost_ray==0.0.1
+zoopt==0.4.1

python/requirements_tune.txt

@@ -1,38 +0,0 @@
-ax-platform
-bayesian-optimization
-ConfigSpace==0.4.10
-dragonfly-opt
-gluoncv
-gorilla # Need this because bug in mlflow. Should be fixed in v1.12.2
-gym[atari]
-GPy
-h5py
-hpbandster
-hyperopt>=0.2.5
-jupyter
-keras
-kubernetes
-lightgbm
-matplotlib
-mlflow
-mxnet
-nevergrad
-optuna
-pytest-remotedata>=0.3.1
-pytorch-lightning==1.0.3
-pytorch-lightning-bolts
-scikit-learn
-scikit-optimize
-sigopt
-smart_open
-tensorflow-probability
-timm
-torch>=1.6.0
-torchvision>=0.6.0
-# transformers
-git+git://github.com/huggingface/transformers.git@bdcc4b78a27775d1ec8f3fd297cb679c257289db#transformers
-git+git://github.com/ray-project/tune-sklearn@master#tune-sklearn
-git+git://github.com/ray-project/xgboost_ray@master#xgboost_ray
-wandb
-xgboost
-zoopt>=0.4.1