From 583df089573a9ddea917710856970a28f7ff1aa8 Mon Sep 17 00:00:00 2001 From: Johann Schleier-Smith Date: Tue, 2 Aug 2016 17:03:28 -0700 Subject: [PATCH] Docker builds on Travis (#343) * attempt to build on travis using docker * run tests in foreground * add examples to travis tests * test from current checkout * attempt to fix docker version issues * try build with xenial * attempt docker upgrade * avoid hang on configuration files * matrix osx and linux w/ docker * restore non-test docker builds * fix typo * tuning and cleanup * add missing file * comment cleanup --- .travis.yml | 18 ++++++------------ docker/deploy/Dockerfile | 3 +++ docker/devel/Dockerfile | 3 +++ docker/examples/Dockerfile | 3 +++ docker/test-base/Dockerfile | 19 +++++++++++++++++++ docker/test-examples/Dockerfile | 10 ++++++++++ test/travis-ci/base_test.sh | 14 ++++++++++++++ test/travis-ci/hyperopt_example.sh | 7 +++++++ test/travis-ci/install.sh | 28 ++++++++++++++++++++++++++++ test/travis-ci/lbfgs_example.sh | 7 +++++++ 10 files changed, 100 insertions(+), 12 deletions(-) create mode 100644 docker/test-base/Dockerfile create mode 100644 docker/test-examples/Dockerfile create mode 100755 test/travis-ci/base_test.sh create mode 100755 test/travis-ci/hyperopt_example.sh create mode 100755 test/travis-ci/install.sh create mode 100755 test/travis-ci/lbfgs_example.sh diff --git a/.travis.yml b/.travis.yml index c87e4c284..c8a028df4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,19 +9,13 @@ matrix: - os: osx osx_image: xcode7 -before_install: - - if [[ $TRAVIS_OS_NAME == 'linux' ]]; then sudo apt-get -y update ; fi - - if [[ $TRAVIS_OS_NAME == 'linux' ]]; then sudo apt-get -y install git ; fi - - if [[ $TRAVIS_OS_NAME == 'linux' ]]; then sudo add-apt-repository --yes ppa:kalakris/cmake ; fi +services: + - docker install: - - ./install-dependencies.sh - - ./setup.sh - - ./build.sh + - ./test/travis-ci/install.sh script: - - source setup-env.sh - - cd test - - python runtest.py - - python array_test.py - - python microbenchmarks.py + - ./test/travis-ci/base_test.sh + - ./test/travis-ci/hyperopt_example.sh + - ./test/travis-ci/lbfgs_example.sh diff --git a/docker/deploy/Dockerfile b/docker/deploy/Dockerfile index dfb278ca9..63ca1337c 100644 --- a/docker/deploy/Dockerfile +++ b/docker/deploy/Dockerfile @@ -1,3 +1,6 @@ +# The deploy Docker image build a self-contained Ray instance suitable +# for end users. + FROM ubuntu:xenial RUN apt-get update RUN apt-get -y install apt-utils diff --git a/docker/devel/Dockerfile b/docker/devel/Dockerfile index bd6c22f79..cdabefcc2 100644 --- a/docker/devel/Dockerfile +++ b/docker/devel/Dockerfile @@ -1,3 +1,6 @@ +# The devel Docker image is designed for use with a source checkout +# mounted from the local host filesystem. + FROM ubuntu:xenial RUN apt-get update RUN apt-get -y install apt-utils diff --git a/docker/examples/Dockerfile b/docker/examples/Dockerfile index 01c9f9283..1b5d5b997 100644 --- a/docker/examples/Dockerfile +++ b/docker/examples/Dockerfile @@ -1,3 +1,6 @@ +# Bulding on top of deploy image, this Dockerfile adds libraries needed +# for running examples. + FROM amplab/ray:deploy # Tensorflow diff --git a/docker/test-base/Dockerfile b/docker/test-base/Dockerfile new file mode 100644 index 000000000..b1c1f740c --- /dev/null +++ b/docker/test-base/Dockerfile @@ -0,0 +1,19 @@ +# Base image for tests. This differs from the deploy image in that +# rather than downloading source code from github it instead adds +# it from a tar file creaetd by test/travis-ci/install.sh + +FROM ubuntu:xenial +RUN apt-get update +RUN apt-get -y install apt-utils +RUN apt-get -y install sudo +RUN apt-get install -y git cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip graphviz +RUN pip install ipython typing funcsigs subprocess32 protobuf==3.0.0a2 colorama graphviz cloudpickle +RUN adduser --gecos --ingroup ray-user --disabled-login --gecos ray-user +RUN adduser ray-user sudo +RUN sed -i "s|%sudo\tALL=(ALL:ALL) ALL|%sudo\tALL=NOPASSWD: ALL|" /etc/sudoers +ADD ray.tar /home/ray-user/ray +RUN chown -R ray-user.ray-user /home/ray-user/ray +USER ray-user +WORKDIR /home/ray-user/ray +RUN ./setup.sh +RUN ./build.sh diff --git a/docker/test-examples/Dockerfile b/docker/test-examples/Dockerfile new file mode 100644 index 000000000..92af99829 --- /dev/null +++ b/docker/test-examples/Dockerfile @@ -0,0 +1,10 @@ +# Bulding on top of base test image, this Dockerfile adds libraries +# needed for running additional examples. + +FROM amplab/ray:test-base + +# Tensorflow +RUN pip install --upgrade https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-0.9.0-cp27-none-linux_x86_64.whl + +# SciPy +RUN pip install scipy diff --git a/test/travis-ci/base_test.sh b/test/travis-ci/base_test.sh new file mode 100755 index 000000000..14fd9e50e --- /dev/null +++ b/test/travis-ci/base_test.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +if [[ $TRAVIS_OS_NAME == 'linux' ]]; then + # Linux test uses Docker + docker run amplab/ray:test-base bash -c 'source setup-env.sh && cd test && python runtest.py && python array_test.py && python microbenchmarks.py' +else + # Mac OS X test + source setup-env.sh + pushd test + runtest.py + python array_test.py + python microbenchmarks.py + popd +fi diff --git a/test/travis-ci/hyperopt_example.sh b/test/travis-ci/hyperopt_example.sh new file mode 100755 index 000000000..9787cf17a --- /dev/null +++ b/test/travis-ci/hyperopt_example.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Hyperparameter optimization test +# Runs only on Docker under Linux +if [[ $TRAVIS_OS_NAME == 'linux' ]]; then + docker run --shm-size=500m amplab/ray:test-examples bash -c 'source setup-env.sh && cd examples/hyperopt && python driver.py' +fi diff --git a/test/travis-ci/install.sh b/test/travis-ci/install.sh new file mode 100755 index 000000000..b6fb5d472 --- /dev/null +++ b/test/travis-ci/install.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +if [[ $TRAVIS_OS_NAME == 'linux' ]]; then + # Linux test uses Docker + + # We need to update the Docker version provided by Travis CI + # in order to set shm size, a setting required by some of the + # tests. + sudo apt-get update + sudo apt-get -y install apt-transport-https ca-certificates + sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D + echo deb https://apt.dockerproject.org/repo ubuntu-trusty main | sudo tee --append /etc/apt/sources.list.d/docker.list + sudo apt-get update + sudo apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install docker-engine + docker version + + # We tar the current checkout, then include it in the Docker image + tar --exclude './docker' -cv . > ./docker/test-base/ray.tar + docker build --no-cache -t amplab/ray:test-base docker/test-base + rm ./docker/test-base/ray.tar + docker build --no-cache -t amplab/ray:test-examples docker/test-examples + docker ps -a +else + # Mac OS X test + ./install-dependencies.sh + ./setup.sh + ./build.sh +fi diff --git a/test/travis-ci/lbfgs_example.sh b/test/travis-ci/lbfgs_example.sh new file mode 100755 index 000000000..5fff50480 --- /dev/null +++ b/test/travis-ci/lbfgs_example.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# L-BFGS Test +# Runs only on Docker under Linux +if [[ $TRAVIS_OS_NAME == 'linux' ]]; then + docker run --shm-size=500m amplab/ray:test-examples bash -c 'source setup-env.sh && cd examples/lbfgs && python driver.py' +fi