Do not compile redis twice (#5074)

ci/travis/install-with-cache-libs.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-
-# Cause the script to exit if a single command fails.
-set -e
-
-ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
-
-# Copy the libs to thirdparty/external_project_libs.
-# -n means do not build again.
-bash $ROOT_DIR/../../thirdparty/scripts/collect_dependent_libs.sh -n
-
-# Import all the libs cached in local to environment variables.
-source $ROOT_DIR/../../thirdparty/external_project_libs/resource.txt
-
-# Clean the arrow building libs and use the cache libs.
-pushd $ROOT_DIR/../../build/external/arrow/src/arrow_ep
-echo "Clean Arrow building files."
-rm -rf $ROOT_DIR/../../build/external/arrow/src/arrow_ep-build/
-popd
-
-# Rebuild ray with libs cache environment variables, which is fast.
-bash $ROOT_DIR/install-ray.sh

thirdparty/scripts/build_arrow.sh

@@ -1,164 +0,0 @@
-#!/bin/bash
-
-set -x
-
-# Cause the script to exit if a single command fails.
-set -e
-
-TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../
-
-# setup env
-if [[ -z  "$1" ]]; then
-  PYTHON_EXECUTABLE=`which python`
-else
-  PYTHON_EXECUTABLE=$1
-fi
-echo "Using Python executable $PYTHON_EXECUTABLE."
-
-unamestr="$(uname)"
-
-if [[ "$unamestr" == "Linux" ]]; then
-  FLATBUFFERS_HOME=$TP_DIR/pkg/flatbuffers
-else
-  FLATBUFFERS_HOME=""
-fi
-
-# Determine how many parallel jobs to use for make based on the number of cores
-if [[ "$unamestr" == "Linux" ]]; then
-  PARALLEL=$(nproc)
-elif [[ "$unamestr" == "Darwin" ]]; then
-  PARALLEL=$(sysctl -n hw.ncpu)
-  echo "Platform is macosx."
-else
-  echo "Unrecognized platform."
-  exit 1
-fi
-
-# The PR for this commit is https://github.com/apache/arrow/pull/2482. We
-# include the link here to make it easier to find the right commit because
-# Arrow often rewrites git history and invalidates certain commits.
-TARGET_COMMIT_ID=927bd34aaad875e82beca2584d5d777839fa8bb0
-build_arrow() {
-  echo "building arrow"
-  # Make sure arrow will be built again when building ray for java later than python
-  if [[ "$RAY_BUILD_JAVA" == "YES" ]]; then
-    rm -rf $TP_DIR/build/arrow/cpp/build/CMakeCache.txt
-  fi
-
-  if [[ ! -d $TP_DIR/build/arrow/.git ]]; then
-    if [[ -d $TP_DIR/build/arrow ]]; then
-      rm -rf $TP_DIR/build/arrow
-    fi
-    git clone -q https://github.com/apache/arrow.git "$TP_DIR/build/arrow"
-  fi
-
-  if ! [ -x "$(command -v bison)" ]; then
-    echo 'Error: bison is not installed.' >&2
-    exit 1
-  fi
-
-  if ! [ -x "$(command -v flex)" ]; then
-    echo 'Error: flex is not installed.' >&2
-    exit 1
-  fi
-
-  pushd $TP_DIR/build/arrow
-  git fetch origin master
-
-  git checkout $TARGET_COMMIT_ID
-
-  cd cpp
-  if [ ! -d "build" ]; then
-    mkdir build
-  fi
-  cd build
-
-  BUILD_ARROW_PLASMA_JAVA_CLIENT=off
-  if [[ "$RAY_BUILD_JAVA" == "YES" ]]; then
-    BUILD_ARROW_PLASMA_JAVA_CLIENT=on
-  fi
-
-  # Clean the build cache for arrow and parquet, in case the error of "Cannot find Parquet" occurs.
-  rm -rf $TP_DIR/build/arrow/python/build/temp*
-
-  ARROW_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install
-  BOOST_ROOT=$TP_DIR/pkg/boost \
-  FLATBUFFERS_HOME=$FLATBUFFERS_HOME \
-  cmake -DCMAKE_BUILD_TYPE=Release \
-      -DCMAKE_C_FLAGS="-g -O3" \
-      -DCMAKE_CXX_FLAGS="-g -O3" \
-      -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
-      -DARROW_BUILD_TESTS=off \
-      -DARROW_HDFS=on \
-      -DARROW_BOOST_USE_SHARED=off \
-      -DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE \
-      -DARROW_PYTHON=on \
-      -DARROW_PLASMA=on \
-      -DARROW_TENSORFLOW=on \
-      -DARROW_JEMALLOC=off \
-      -DARROW_WITH_BROTLI=off \
-      -DARROW_WITH_LZ4=off \
-      -DARROW_WITH_ZLIB=off \
-      -DARROW_WITH_ZSTD=off \
-      -DARROW_PLASMA_JAVA_CLIENT=$BUILD_ARROW_PLASMA_JAVA_CLIENT \
-      ..
-  make VERBOSE=1 -j$PARALLEL
-  make install
-
-  if [[ -d $ARROW_HOME/lib64 ]]; then
-      # On CentOS, Arrow gets installed under lib64 instead of lib, so copy it for
-      # now. TODO(rkn): A preferable solution would be to add both directories to
-      # the PKG_CONFIG_PATH, but that didn't seem to work.
-      cp -r $ARROW_HOME/lib64 $ARROW_HOME/lib
-  fi
-
-  bash "$TP_DIR/scripts/build_parquet.sh"
-
-  echo "installing pyarrow"
-  cd $TP_DIR/build/arrow/python
-  # We set PKG_CONFIG_PATH, which is important so that in cmake, pkg-config can
-  # find plasma.
-  PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
-  PYARROW_WITH_PLASMA=1 \
-  PYARROW_WITH_TENSORFLOW=1 \
-  PYARROW_BUNDLE_ARROW_CPP=1 \
-  $PYTHON_EXECUTABLE setup.py build
-
-  PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
-  PYARROW_WITH_PLASMA=1 \
-  PYARROW_WITH_TENSORFLOW=1 \
-  PYARROW_BUNDLE_ARROW_CPP=1 \
-  PARQUET_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install \
-  PYARROW_WITH_PARQUET=1 \
-  $PYTHON_EXECUTABLE setup.py build_ext
-
-  # Find the pyarrow directory that was just built and copy it to ray/python/ray/
-  # so that pyarrow can be packaged along with ray.
-  pushd $TP_DIR/build/arrow/python/build
-  PYARROW_BUILD_LIB_DIR="$TP_DIR/build/arrow/python/build/$(find ./ -maxdepth 1 -type d -print | grep -m1 'lib')"
-  popd
-  echo "copying pyarrow files from $PYARROW_BUILD_LIB_DIR/pyarrow"
-  cp -r $PYARROW_BUILD_LIB_DIR/pyarrow $TP_DIR/../python/ray/pyarrow_files/
-
-  popd
-}
-# Download and compile arrow if it isn't already present or the commit-id mismatches.
-if [[ "$RAY_BUILD_PYTHON" == "YES" && ! -d $TP_DIR/../python/ray/pyarrow_files/pyarrow ]] || \
-    [[ "$RAY_BUILD_JAVA" == "YES" && ! -f $TP_DIR/build/arrow/cpp/build/release/libplasma_java.dylib ]]; then
-  build_arrow
-else
-  REBUILD=off
-  pushd $TP_DIR/build/arrow
-  if [[ "$TARGET_COMMIT_ID" != `git rev-parse HEAD` ]]; then
-    # TARGET_COMMIT_ID may change to later commit.
-    echo "Commit ID mismatches."
-    git fetch origin master
-    git checkout $TARGET_COMMIT_ID
-    REBUILD=on
-  fi
-  popd
-
-  if [[ "$REBUILD" == "on" ]]; then
-    build_arrow
-  fi
-fi

thirdparty/scripts/build_boost.sh

@@ -1,33 +0,0 @@
-#!/bin/bash
-
-set -x
-
-# Cause the script to exit if a single command fails.
-set -e
-
-TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../
-
-BOOST_VERSION=1.65.1
-BOOST_VERSION_UNDERSCORE=1_65_1
-
-# Download and compile boost if it isn't already present.
-if [[ ! -d $TP_DIR/pkg/boost ]]; then
-  # The wget command frequently fails, so retry up to 20 times.
-  for COUNT in {1..20}; do
-    # Attempt to wget boost and break from the retry loop if it succeeds.
-    wget --progress=dot:giga --no-check-certificate http://dl.bintray.com/boostorg/release/$BOOST_VERSION/source/boost_$BOOST_VERSION_UNDERSCORE.tar.gz -O $TP_DIR/build/boost_$BOOST_VERSION_UNDERSCORE.tar.gz && break
-    # If none of the retries succeeded at getting boost, then fail.
-    if [[ $COUNT == 20 ]]; then
-      exit 1
-    fi
-  done
-
-  tar xf $TP_DIR/build/boost_$BOOST_VERSION_UNDERSCORE.tar.gz -C $TP_DIR/build/
-  rm -rf $TP_DIR/build/boost_$BOOST_VERSION_UNDERSCORE.tar.gz
-
-  # Compile boost.
-  pushd $TP_DIR/build/boost_$BOOST_VERSION_UNDERSCORE
-    ./bootstrap.sh
-    ./bjam cxxflags=-fPIC cflags=-fPIC variant=release link=static --prefix=$TP_DIR/pkg/boost --with-filesystem --with-system --with-thread --with-regex install > /dev/null
-  popd
-fi

thirdparty/scripts/build_credis.sh

@@ -1,74 +0,0 @@
-#!/usr/bin/env bash
-
-# Usage: by default jemalloc is used; however, if tcmalloc is installed on the
-# system, credis' CMakeLists.txt will prefer it over jemalloc.  To avoid build
-# failures use:
-#
-#     CREDIS_TCMALLOC=1 build_credis.sh
-#
-# If building all of ray from ray/python, use:
-#
-#     env CREDIS_TCMALLOC=1 RAY_USE_NEW_GCS=on pip install -e . --verbose
-
-set -x
-
-# Cause the script to exit if a single command fails.
-set -e
-
-TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../
-ROOT_DIR=$TP_DIR/..
-
-# For some reason, on Ubuntu/gcc toolchain linking against static libleveldb.a
-# doesn't work, so we force building the shared library for non-Mac.
-if [ "$(uname)" == "Darwin" ]; then
-    BUILD_LEVELDB_CONFIG=""
-else
-    BUILD_LEVELDB_CONFIG="-DBUILD_SHARED_LIBS=on"
-fi
-
-if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then
-  pushd "$TP_DIR/pkg/"
-    if [[ ! -d "credis" ]]; then
-      git clone -q --recursive https://github.com/ray-project/credis
-    fi
-  popd
-
-  pushd "$TP_DIR/pkg/credis"
-    git fetch origin master
-    git checkout 273d667e5126c246b45f5dcf030b651a653136c3
-
-    # If the above commit points to different submodules' commits than
-    # origin's head, this updates the submodules.
-    git submodule update
-
-    # TODO(pcm): Get the build environment for tcmalloc set up and compile redis
-    # with tcmalloc.
-    # NOTE(zongheng): if we don't specify MALLOC=jemalloc, then build behiavors
-    # differ between Mac (libc) and Linux (jemalloc)... This breaks our CMake
-    # rules.
-    if [[ "${CREDIS_TCMALLOC}" = 1 ]]; then
-        echo "CREDIS_MALLOC is set, using tcmalloc to build redis"
-        pushd redis && env USE_TCMALLOC=yes make -j && popd
-    else
-        pushd redis && make -j MALLOC=jemalloc && popd
-    fi
-    pushd glog; cmake -DWITH_GFLAGS=off . && make -j; popd
-    pushd leveldb;
-      mkdir -p build && cd build
-      cmake ${BUILD_LEVELDB_CONFIG} -DCMAKE_BUILD_TYPE=Release .. && cmake --build .
-    popd
-
-    mkdir -p build
-    pushd build
-      cmake -DCMAKE_BUILD_TYPE=Release ..
-      make -j
-    popd
-
-    mkdir -p $ROOT_DIR/build/src/credis/redis/src/
-    cp redis/src/redis-server $ROOT_DIR/build/src/credis/redis/src/redis-server
-
-    mkdir -p $ROOT_DIR/build/src/credis/build/src/
-    cp build/src/libmaster.so $ROOT_DIR/build/src/credis/build/src/libmaster.so
-    cp build/src/libmember.so $ROOT_DIR/build/src/credis/build/src/libmember.so
-  popd
-fi

thirdparty/scripts/build_flatbuffers.sh

@@ -1,30 +0,0 @@
-#!/bin/bash
-
-set -x
-
-# Cause the script to exit if a single command fails.
-set -e
-
-TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../
-
-FLATBUFFERS_VERSION=1.9.0
-
-# Download and compile flatbuffers if it isn't already present.
-if [ ! -d $TP_DIR/pkg/flatbuffers ]; then
-  echo "building flatbuffers"
-  pushd $TP_DIR/build
-  curl -sL https://github.com/google/flatbuffers/archive/v$FLATBUFFERS_VERSION.tar.gz -o flatbuffers-$FLATBUFFERS_VERSION.tar.gz
-  tar xf flatbuffers-$FLATBUFFERS_VERSION.tar.gz
-  rm -rf flatbuffers-$FLATBUFFERS_VERSION.tar.gz
-
-  # Compile flatbuffers.
-  pushd flatbuffers-$FLATBUFFERS_VERSION
-    cmake -DCMAKE_CXX_FLAGS=-fPIC \
-          -DCMAKE_BUILD_TYPE=Release \
-          -DCMAKE_INSTALL_PREFIX:PATH=$TP_DIR/pkg/flatbuffers \
-          -DFLATBUFFERS_BUILD_TESTS=OFF
-    make -j5
-    make install
-  popd
-  popd
-fi

thirdparty/scripts/build_redis.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-
-set -x
-
-# Cause the script to exit if a single command fails.
-set -e
-
-TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../
-
-if [ ! -f $TP_DIR/pkg/redis/src/redis-server ]; then
-  redis_vname="5.0.3"
-  # This check is to make sure the tarball has been fully extracted. The only
-  # relevant bit about redis/utils/whatisdoing.sh is that it is one of the last
-  # files in the tarball.
-  if [ ! -f $TP_DIR/pkg/redis/utils/whatisdoing.sh ]; then
-    mkdir -p "$TP_DIR/pkg/redis"
-    curl -sL "https://github.com/antirez/redis/archive/$redis_vname.tar.gz" | tar xz --strip-components=1 -C "$TP_DIR/pkg/redis"
-  fi
-  pushd $TP_DIR/pkg/redis
-  make
-  popd
-fi

thirdparty/scripts/collect_dependent_libs.sh

@@ -1,116 +0,0 @@
-#!/usr/bin/env bash
-set -x
-
-# Cause the script to exit if a single command fails.
-set -e
-
-ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
-
-function usage() {
-  echo "Usage: collect_dependent_libs.sh [<args>]"
-  echo
-  echo "Options:"
-  echo "  -h|--help               print the help info"
-  echo "  -d|--target-dir         the target directory to put all the thirdparty libs"
-  echo "  -n|--no-build           do not build ray, used in case that ray is already built"
-  echo "  -r|--resource           the resource file name (default: resource.txt)"
-  echo
-}
-
-# By default all the libs will be put into ./thirdparty/external_project_libs.
-# However, this directory could be cleaned by `git clean`.
-# Users can provide another directory using -d option.
-DIR="$ROOT_DIR/../external_project_libs"
-# By default ray will be built before copying the libs.
-# Users can skip the building process if they have built ray.
-BUILD="YES"
-
-RESOURCE="resource.txt"
-
-# Parse options
-while [[ $# > 0 ]]; do
-  key="$1"
-  case $key in
-    -h|--help)
-      usage
-      exit 0
-      ;;
-    -d|--target-dir)
-      DIR="$2"
-      shift
-      ;;
-    -n|--no-build)
-      BUILD="NO"
-      ;;
-    -r|--resource)
-      RESOURCE="$2"
-      shift
-      ;;
-    *)
-      echo "ERROR: unknown option \"$key\""
-      echo
-      usage
-      exit -1
-      ;;
-  esac
-  shift
-done
-
-echo "External project libs will be put to $DIR"
-if [ ! -d "$DIR" ]; then
-  mkdir -p $DIR
-fi
-
-pushd $ROOT_DIR
-if [ "$BUILD" = "YES" ]; then
-  echo "Build Ray First."
-  ../../build.sh
-fi
-
-RAY_BUILD_DIR=$ROOT_DIR/../../build/external/
-ARROW_BUILD_DIR=$ROOT_DIR/../../build/external/arrow/src/arrow_ep-build/
-
-function cp_one_lib() {
-  if [[ ! -d "$1" ]]; then
-    echo "Lib root dir $1 does not exist!"
-    exit -1
-  fi
-  if [[ ! -d "$1/include" ]]; then
-    echo "Lib inlcude dir $1 does not exist!"
-    exit -1
-  fi
-  if [[ ! -d "$1/lib" && ! -d "$1/lib64" ]]; then
-    echo "Lib dir $1 does not exist!"
-    exit -1
-  fi
-  cp -rf $1 $DIR
-}
-
-# copy libs that ray needs.
-cp_one_lib $RAY_BUILD_DIR/boost-install
-cp_one_lib $RAY_BUILD_DIR/flatbuffers-install
-cp_one_lib $RAY_BUILD_DIR/glog-install
-cp_one_lib $RAY_BUILD_DIR/googletest-install
-
-# copy libs that arrow needs.
-cp_one_lib $ARROW_BUILD_DIR/snappy_ep/src/snappy_ep-install
-cp_one_lib $ARROW_BUILD_DIR/thrift_ep/src/thrift_ep-install
-
-# generate the export script.
-echo "Output the exporting resource file to $DIR/$RESOURCE."
-echo "export BOOST_ROOT=$DIR/boost-install" > $DIR/$RESOURCE
-echo "export RAY_BOOST_ROOT=\$BOOST_ROOT" >> $DIR/$RESOURCE
-
-echo "export FLATBUFFERS_HOME=$DIR/flatbuffers-install" >> $DIR/$RESOURCE
-echo "export RAY_FLATBUFFERS_HOME=\$FLATBUFFERS_HOME" >> $DIR/$RESOURCE
-
-echo "export GTEST_HOME=$DIR/googletest-install" >> $DIR/$RESOURCE
-echo "export RAY_GTEST_HOME=\$GTEST_HOME" >> $DIR/$RESOURCE
-
-echo "export GLOG_HOME=$DIR/glog-install" >> $DIR/$RESOURCE
-echo "export RAY_GLOG_HOME=\$GLOG_HOME" >> $DIR/$RESOURCE
-
-echo "export SNAPPY_HOME=$DIR/snappy_ep-install" >> $DIR/$RESOURCE
-echo "export THRIFT_HOME=$DIR/thrift_ep-install" >> $DIR/$RESOURCE
-
-popd

thirdparty/scripts/setup.sh

@@ -27,41 +27,6 @@ fi
 
 unamestr="$(uname)"
 
-##############################################
-# boost
-##############################################
-#bash "$TP_SCRIPT_DIR/build_boost.sh"
-
-##############################################
-# redis
-##############################################
-bash "$TP_SCRIPT_DIR/build_redis.sh"
-
-##############################################
-# credis
-##############################################
-bash "$TP_SCRIPT_DIR/build_credis.sh"
-
-##############################################
-# flatbuffers if necessary
-##############################################
-#if [[ "$unamestr" == "Linux" ]]; then
-#  echo "building flatbuffers"
-#  bash "$TP_SCRIPT_DIR/build_flatbuffers.sh"
-#fi
-
-##############################################
-# arrow
-##############################################
-#RAY_BUILD_PYTHON=$RAY_BUILD_PYTHON \
-#RAY_BUILD_JAVA=$RAY_BUILD_JAVA \
-#bash "$TP_SCRIPT_DIR/build_arrow.sh" $PYTHON_EXECUTABLE
-
-##############################################
-# rDSN (optional)
-##############################################
-# bash "$TP_SCRIPT_DIR/build_rdsn.sh"
-
 ##############################################
 # modin
 ##############################################

thirdparty/scripts/thirdparty.cmake

@@ -1,6 +0,0 @@
-
-# add all thirdparty related path definitions here
-
-list(APPEND CMAKE_MODULE_PATH
-            ${CMAKE_CURRENT_LIST_DIR}/../build/arrow/python/cmake_modules)
-message (WARNING ${CMAKE_MODULE_PATH})