mirror of
https://github.com/vale981/ray
synced 2025-03-06 02:21:39 -05:00
231 lines
8 KiB
Python
231 lines
8 KiB
Python
from itertools import chain
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
|
|
from setuptools import setup, find_packages, Distribution
|
|
import setuptools.command.build_ext as _build_ext
|
|
|
|
# Ideally, we could include these files by putting them in a
|
|
# MANIFEST.in or using the package_data argument to setup, but the
|
|
# MANIFEST.in gets applied at the very beginning when setup.py runs
|
|
# before these files have been created, so we have to move the files
|
|
# manually.
|
|
|
|
exe_suffix = ".exe" if sys.platform == "win32" else ""
|
|
|
|
# .pyd is the extension Python requires on Windows for shared libraries.
|
|
# https://docs.python.org/3/faq/windows.html#is-a-pyd-file-the-same-as-a-dll
|
|
pyd_suffix = ".pyd" if sys.platform == "win32" else ".so"
|
|
|
|
# NOTE: The lists below must be kept in sync with ray/BUILD.bazel.
|
|
ray_files = [
|
|
"ray/core/src/ray/thirdparty/redis/src/redis-server",
|
|
"ray/core/src/ray/gcs/redis_module/libray_redis_module.so",
|
|
"ray/core/src/plasma/plasma_store_server" + exe_suffix,
|
|
"ray/_raylet" + pyd_suffix,
|
|
"ray/core/src/ray/raylet/raylet_monitor" + exe_suffix,
|
|
"ray/core/src/ray/gcs/gcs_server" + exe_suffix,
|
|
"ray/core/src/ray/raylet/raylet" + exe_suffix,
|
|
"ray/streaming/_streaming.so",
|
|
]
|
|
|
|
build_java = os.getenv("RAY_INSTALL_JAVA") == "1"
|
|
if build_java:
|
|
ray_files.append("ray/jars/ray_dist.jar")
|
|
|
|
# These are the directories where automatically generated Python protobuf
|
|
# bindings are created.
|
|
generated_python_directories = [
|
|
"ray/core/generated",
|
|
"ray/streaming/generated",
|
|
]
|
|
|
|
optional_ray_files = []
|
|
|
|
ray_autoscaler_files = [
|
|
"ray/autoscaler/aws/example-full.yaml",
|
|
"ray/autoscaler/azure/example-full.yaml",
|
|
"ray/autoscaler/azure/azure-vm-template.json",
|
|
"ray/autoscaler/azure/azure-config-template.json",
|
|
"ray/autoscaler/gcp/example-full.yaml",
|
|
"ray/autoscaler/local/example-full.yaml",
|
|
"ray/autoscaler/kubernetes/example-full.yaml",
|
|
"ray/autoscaler/kubernetes/kubectl-rsync.sh",
|
|
"ray/autoscaler/ray-schema.json"
|
|
]
|
|
|
|
ray_project_files = [
|
|
"ray/projects/schema.json", "ray/projects/templates/cluster_template.yaml",
|
|
"ray/projects/templates/project_template.yaml",
|
|
"ray/projects/templates/requirements.txt"
|
|
]
|
|
|
|
ray_dashboard_files = [
|
|
os.path.join(dirpath, filename)
|
|
for dirpath, dirnames, filenames in os.walk("ray/dashboard/client/build")
|
|
for filename in filenames
|
|
]
|
|
|
|
optional_ray_files += ray_autoscaler_files
|
|
optional_ray_files += ray_project_files
|
|
optional_ray_files += ray_dashboard_files
|
|
|
|
if "RAY_USE_NEW_GCS" in os.environ and os.environ["RAY_USE_NEW_GCS"] == "on":
|
|
ray_files += [
|
|
"ray/core/src/credis/build/src/libmember.so",
|
|
"ray/core/src/credis/build/src/libmaster.so",
|
|
"ray/core/src/credis/redis/src/redis-server"
|
|
]
|
|
|
|
extras = {
|
|
"debug": [],
|
|
"dashboard": ["requests", "gpustat"],
|
|
"serve": ["uvicorn", "flask", "blist"],
|
|
"tune": ["tabulate", "tensorboardX", "pandas"]
|
|
}
|
|
|
|
extras["rllib"] = extras["tune"] + [
|
|
"atari_py",
|
|
"dm_tree",
|
|
"gym[atari]",
|
|
"lz4",
|
|
"opencv-python-headless",
|
|
"pyyaml",
|
|
"scipy",
|
|
]
|
|
|
|
extras["streaming"] = ["msgpack >= 0.6.2"]
|
|
|
|
extras["all"] = list(set(chain.from_iterable(extras.values())))
|
|
|
|
|
|
class build_ext(_build_ext.build_ext):
|
|
def run(self):
|
|
# Note: We are passing in sys.executable so that we use the same
|
|
# version of Python to build packages inside the build.sh script. Note
|
|
# that certain flags will not be passed along such as --user or sudo.
|
|
# TODO(rkn): Fix this.
|
|
command = ["../build.sh", "-p", sys.executable]
|
|
if sys.platform == "win32" and command[0].lower().endswith(".sh"):
|
|
# We can't run .sh files directly in Windows, so find a shell.
|
|
# Don't use "bash" instead of "sh", because that might run the Bash
|
|
# from WSL! (We want MSYS2's Bash, which is also sh by default.)
|
|
shell = os.getenv("BAZEL_SH", "sh") # NOT "bash"! (see above)
|
|
command.insert(0, shell)
|
|
if build_java:
|
|
# Also build binaries for Java if the above env variable exists.
|
|
command += ["-l", "python,java"]
|
|
subprocess.check_call(command)
|
|
|
|
# We also need to install pickle5 along with Ray, so make sure that the
|
|
# relevant non-Python pickle5 files get copied.
|
|
pickle5_files = self.walk_directory("./ray/pickle5_files/pickle5")
|
|
|
|
thirdparty_files = self.walk_directory("./ray/thirdparty_files")
|
|
|
|
files_to_include = ray_files + pickle5_files + thirdparty_files
|
|
|
|
# Copy over the autogenerated protobuf Python bindings.
|
|
for directory in generated_python_directories:
|
|
for filename in os.listdir(directory):
|
|
if filename[-3:] == ".py":
|
|
files_to_include.append(os.path.join(directory, filename))
|
|
|
|
for filename in files_to_include:
|
|
self.move_file(filename)
|
|
|
|
# Try to copy over the optional files.
|
|
for filename in optional_ray_files:
|
|
try:
|
|
self.move_file(filename)
|
|
except Exception:
|
|
print("Failed to copy optional file {}. This is ok."
|
|
.format(filename))
|
|
|
|
def walk_directory(self, directory):
|
|
file_list = []
|
|
for (root, dirs, filenames) in os.walk(directory):
|
|
for name in filenames:
|
|
file_list.append(os.path.join(root, name))
|
|
return file_list
|
|
|
|
def move_file(self, filename):
|
|
# TODO(rkn): This feels very brittle. It may not handle all cases. See
|
|
# https://github.com/apache/arrow/blob/master/python/setup.py for an
|
|
# example.
|
|
source = filename
|
|
destination = os.path.join(self.build_lib, filename)
|
|
# Create the target directory if it doesn't already exist.
|
|
os.makedirs(os.path.dirname(destination), exist_ok=True)
|
|
if not os.path.exists(destination):
|
|
print("Copying {} to {}.".format(source, destination))
|
|
if sys.platform == "win32":
|
|
# Does not preserve file mode (needed to avoid read-only bit)
|
|
shutil.copyfile(source, destination, follow_symlinks=True)
|
|
else:
|
|
# Preserves file mode (needed to copy executable bit)
|
|
shutil.copy(source, destination, follow_symlinks=True)
|
|
|
|
|
|
class BinaryDistribution(Distribution):
|
|
def has_ext_modules(self):
|
|
return True
|
|
|
|
|
|
def find_version(*filepath):
|
|
# Extract version information from filepath
|
|
here = os.path.abspath(os.path.dirname(__file__))
|
|
with open(os.path.join(here, *filepath)) as fp:
|
|
version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
|
|
fp.read(), re.M)
|
|
if version_match:
|
|
return version_match.group(1)
|
|
raise RuntimeError("Unable to find version string.")
|
|
|
|
|
|
requires = [
|
|
"aiohttp",
|
|
"click >= 7.0",
|
|
"colorama",
|
|
"filelock",
|
|
"google",
|
|
"grpcio",
|
|
"jsonschema",
|
|
"msgpack >= 0.6.0, < 2.0.0",
|
|
"numpy >= 1.16",
|
|
"protobuf >= 3.8.0",
|
|
"py-spy >= 0.2.0",
|
|
"pyyaml",
|
|
"redis >= 3.3.2, < 3.5.0",
|
|
]
|
|
|
|
setup(
|
|
name="ray",
|
|
version=find_version("ray", "__init__.py"),
|
|
author="Ray Team",
|
|
author_email="ray-dev@googlegroups.com",
|
|
description=("A system for parallel and distributed Python that unifies "
|
|
"the ML ecosystem."),
|
|
long_description=open("../README.rst").read(),
|
|
url="https://github.com/ray-project/ray",
|
|
keywords=("ray distributed parallel machine-learning "
|
|
"reinforcement-learning deep-learning python"),
|
|
packages=find_packages(),
|
|
cmdclass={"build_ext": build_ext},
|
|
# The BinaryDistribution argument triggers build_ext.
|
|
distclass=BinaryDistribution,
|
|
install_requires=requires,
|
|
setup_requires=["cython >= 0.29.14", "wheel"],
|
|
extras_require=extras,
|
|
entry_points={
|
|
"console_scripts": [
|
|
"ray=ray.scripts.scripts:main",
|
|
"rllib=ray.rllib.scripts:cli [rllib]", "tune=ray.tune.scripts:cli"
|
|
]
|
|
},
|
|
include_package_data=True,
|
|
zip_safe=False,
|
|
license="Apache 2.0")
|