hiro/ray
1
0
Fork 0
mirror of https://github.com/vale981/ray synced 2025-03-06 02:21:39 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

Update arrow to reduce plasma IPCs. (#3497)

Browse source
This commit is contained in:
Philipp Moritz 2018-12-14 20:49:37 -08:00 • committed by Robert Nishihara
parent fcc37021b2
commit b3bf608608
18 changed files with 27 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.travis/install-dependencies.sh
View file

@ -24,7 +24,7 @@ if [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "linux" ]]; then
wget https://repo.continuum.io/miniconda/Miniconda2-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
pip install -q cython==0.29.0 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
feather-format lxml openpyxl xlrd py-spy setproctitle faulthandler pytest-timeout mock
elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "linux" ]]; then
sudo apt-get update
@ -33,7 +33,7 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "linux" ]]; then
wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
pip install -q cython==0.29.0 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
feather-format lxml openpyxl xlrd py-spy setproctitle pytest-timeout
elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then
# check that brew is installed
@ -50,7 +50,7 @@ elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then
wget https://repo.continuum.io/miniconda/Miniconda2-4.5.4-MacOSX-x86_64.sh -O miniconda.sh -nv
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
pip install -q cython==0.29.0 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
feather-format lxml openpyxl xlrd py-spy setproctitle faulthandler pytest-timeout mock
elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then
# check that brew is installed
@ -67,7 +67,7 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then
wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-MacOSX-x86_64.sh -O miniconda.sh -nv
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
pip install -q cython==0.27.3 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
pip install -q cython==0.29.0 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
feather-format lxml openpyxl xlrd py-spy setproctitle pytest-timeout
elif [[ "$LINT" == "1" ]]; then
sudo apt-get update

4
cmake/Modules/ArrowExternalProject.cmake
View file

@ -15,10 +15,10 @@
# - PLASMA_SHARED_LIB
set(arrow_URL https://github.com/apache/arrow.git)
# The PR for this commit is https://github.com/apache/arrow/pull/3117. We
# The PR for this commit is https://github.com/apache/arrow/pull/3124. We
# include the link here to make it easier to find the right commit because
# Arrow often rewrites git history and invalidates certain commits.
set(arrow_TAG 4d810b4a9c37e79fde6b134ac90ee0c5f7f6c9bf)
set(arrow_TAG b3bc3384f3068edebe69f1084518ccfb85a368f8)
set(ARROW_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/arrow-install)
set(ARROW_HOME ${ARROW_INSTALL_PREFIX})

2
docker/base-deps/Dockerfile
View file

@ -26,6 +26,6 @@ RUN apt-get update \
&& /opt/conda/bin/conda clean -y --all \
&& /opt/conda/bin/pip install \
flatbuffers \
cython==0.27.3
cython==0.29.0
ENV PATH "/opt/conda/bin:$PATH"

4
java/doc/installation.rst
View file

@ -26,7 +26,7 @@ For Ubuntu users, run the following commands:
# If you are on Ubuntu 14.04, you need the following.
pip install cmake
pip install cython==0.27.3
pip install cython==0.29.0
For macOS users, run the following commands:
::
@ -34,7 +34,7 @@ For macOS users, run the following commands:
brew update
brew install maven cmake pkg-config automake autoconf libtool openssl bison wget
pip install cython==0.27.3
pip install cython==0.29.0
Build Ray
^^^^^^^^^

2
python/build-wheel-macos.sh
View file

@ -70,7 +70,7 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
$PIP_CMD install -q setuptools_scm==2.1.0
# Fix the numpy version because this will be the oldest numpy version we can
# support.
$PIP_CMD install -q numpy==$NUMPY_VERSION cython==0.27.3
$PIP_CMD install -q numpy==$NUMPY_VERSION cython==0.29.0
# Install wheel to avoid the error "invalid command 'bdist_wheel'".
$PIP_CMD install -q wheel
# Add the correct Python to the path and build the wheel. This is only

2
python/build-wheel-manylinux1.sh
View file

@ -21,7 +21,7 @@ for PYTHON in cp27-cp27mu cp34-cp34m cp35-cp35m cp36-cp36m cp37-cp37m; do
pushd python
# Fix the numpy version because this will be the oldest numpy version we can
# support.
/opt/python/${PYTHON}/bin/pip install -q numpy==1.10.4 cython==0.27.3
/opt/python/${PYTHON}/bin/pip install -q numpy==1.10.4 cython==0.29.0
INCLUDE_UI=1 PATH=/opt/python/${PYTHON}/bin:$PATH /opt/python/${PYTHON}/bin/python setup.py bdist_wheel
# In the future, run auditwheel here.
mv dist/*.whl ../.whl/

2
python/ray/autoscaler/aws/development-example.yaml
View file

@ -94,7 +94,7 @@ setup_commands:
- echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc
# Build Ray.
- git clone https://github.com/ray-project/ray || true
- pip install boto3==1.4.8 cython==0.27.3
- pip install boto3==1.4.8 cython==0.29.0
- cd ray/python; pip install -e . --verbose
# Custom commands that will be run on the head node after common setup.

2
python/ray/autoscaler/gcp/example-full.yaml
View file

@ -123,7 +123,7 @@ setup_commands:
- >-
pip install
google-api-python-client==1.6.7
cython==0.27.3
cython==0.29.0
# - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.6.0-cp27-cp27mu-manylinux1_x86_64.whl
# - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.6.0-cp35-cp35m-manylinux1_x86_64.whl
# - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.6.0-cp36-cp36m-manylinux1_x86_64.whl

2
python/ray/worker.py
View file

@ -2061,7 +2061,7 @@ def connect(info,
# Create an object store client.
worker.plasma_client = thread_safe_client(
plasma.connect(info["store_socket_name"], "", 64))
plasma.connect(info["store_socket_name"], ""))
raylet_socket = info["raylet_socket_name"]

2
python/setup.py
View file

@ -165,7 +165,7 @@ setup(
# The BinaryDistribution argument triggers build_ext.
distclass=BinaryDistribution,
install_requires=requires,
setup_requires=["cython >= 0.27, < 0.28"],
setup_requires=["cython >= 0.29"],
extras_require=extras,
entry_points={
"console_scripts": [

4
src/ray/object_manager/object_buffer_pool.cc
View file

@ -3,10 +3,10 @@
namespace ray {
ObjectBufferPool::ObjectBufferPool(const std::string &store_socket_name,
uint64_t chunk_size, int release_delay)
uint64_t chunk_size)
: default_chunk_size_(chunk_size) {
store_socket_name_ = store_socket_name;
ARROW_CHECK_OK(store_client_.Connect(store_socket_name_.c_str(), "", release_delay));
ARROW_CHECK_OK(store_client_.Connect(store_socket_name_.c_str(), ""));
}
ObjectBufferPool::~ObjectBufferPool() {

5
src/ray/object_manager/object_buffer_pool.h
View file

@ -40,10 +40,7 @@ class ObjectBufferPool {
/// \param store_socket_name The socket name of the store to which plasma clients
/// connect.
/// \param chunk_size The chunk size into which objects are to be split.
/// \param release_delay The number of release calls before objects are released
/// from the store client (FIFO).
ObjectBufferPool(const std::string &store_socket_name, const uint64_t chunk_size,
const int release_delay);
ObjectBufferPool(const std::string &store_socket_name, const uint64_t chunk_size);
~ObjectBufferPool();

5
src/ray/object_manager/object_manager.cc
View file

@ -14,10 +14,7 @@ ObjectManager::ObjectManager(asio::io_service &main_service,
: config_(config),
object_directory_(std::move(object_directory)),
store_notification_(main_service, config_.store_socket_name),
// release_delay of 2 * config_.max_sends is to ensure the pool does not release
// an object prematurely whenever we reach the maximum number of sends.
buffer_pool_(config_.store_socket_name, config_.object_chunk_size,
/*release_delay=*/2 * config_.max_sends),
buffer_pool_(config_.store_socket_name, config_.object_chunk_size),
send_work_(send_service_),
receive_work_(receive_service_),
connection_pool_(),

3
src/ray/object_manager/object_store_notification_manager.cc
View file

@ -15,8 +15,7 @@ namespace ray {
ObjectStoreNotificationManager::ObjectStoreNotificationManager(
boost::asio::io_service &io_service, const std::string &store_socket_name)
: store_client_(), socket_(io_service) {
ARROW_CHECK_OK(store_client_.Connect(store_socket_name.c_str(), "",
plasma::kPlasmaDefaultReleaseDelay));
ARROW_CHECK_OK(store_client_.Connect(store_socket_name.c_str(), ""));
ARROW_CHECK_OK(store_client_.Subscribe(&c_socket_));
boost::system::error_code ec;

4
src/ray/object_manager/test/object_manager_stress_test.cc
View file

@ -154,8 +154,8 @@ class TestObjectManagerBase : public ::testing::Test {
server2.reset(new MockServer(main_service, om_config_2, gcs_client_2));
// connect to stores.
ARROW_CHECK_OK(client1.Connect(store_id_1, "", plasma::kPlasmaDefaultReleaseDelay));
ARROW_CHECK_OK(client2.Connect(store_id_2, "", plasma::kPlasmaDefaultReleaseDelay));
ARROW_CHECK_OK(client1.Connect(store_id_1, ""));
ARROW_CHECK_OK(client2.Connect(store_id_2, ""));
}
void TearDown() {

4
src/ray/object_manager/test/object_manager_test.cc
View file

@ -139,8 +139,8 @@ class TestObjectManagerBase : public ::testing::Test {
server2.reset(new MockServer(main_service, om_config_2, gcs_client_2));
// connect to stores.
ARROW_CHECK_OK(client1.Connect(store_id_1, "", plasma::kPlasmaDefaultReleaseDelay));
ARROW_CHECK_OK(client2.Connect(store_id_2, "", plasma::kPlasmaDefaultReleaseDelay));
ARROW_CHECK_OK(client1.Connect(store_id_1, ""));
ARROW_CHECK_OK(client2.Connect(store_id_2, ""));
}
void TearDown() {

4
src/ray/raylet/object_manager_integration_test.cc
View file

@ -74,8 +74,8 @@ class TestObjectManagerBase : public ::testing::Test {
GetNodeManagerConfig("raylet_2", store_sock_2), om_config_2, gcs_client_2));
// connect to stores.
ARROW_CHECK_OK(client1.Connect(store_sock_1, "", plasma::kPlasmaDefaultReleaseDelay));
ARROW_CHECK_OK(client2.Connect(store_sock_2, "", plasma::kPlasmaDefaultReleaseDelay));
ARROW_CHECK_OK(client1.Connect(store_sock_1, ""));
ARROW_CHECK_OK(client2.Connect(store_sock_2, ""));
}
void TearDown() {

2
test/stress_tests/stress_testing_config.yaml
View file

@ -94,7 +94,7 @@ setup_commands:
- echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc
# # Build Ray.
# - git clone https://github.com/ray-project/ray || true
- pip install boto3==1.4.8 cython==0.27.3
- pip install boto3==1.4.8 cython==0.29.0
# - cd ray/python; git checkout master; git pull; pip install -e . --verbose
- pip install https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.6.0-cp36-cp36m-manylinux1_x86_64.whl