# Bazel build # C/C++ documentation: https://docs.bazel.build/versions/master/be/c-cpp.html load("@rules_proto//proto:defs.bzl", "proto_library") load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library", "cc_test") load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library") load("@com_github_grpc_grpc//bazel:cython_library.bzl", "pyx_library") load("@rules_proto_grpc//python:defs.bzl", "python_grpc_compile") load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library") load("//bazel:ray.bzl", "COPTS", "PYX_COPTS", "PYX_SRCS") # === Begin of protobuf definitions === proto_library( name = "common_proto", srcs = ["src/ray/protobuf/common.proto"], visibility = ["//java:__subpackages__"], ) cc_proto_library( name = "common_cc_proto", deps = [":common_proto"], ) python_grpc_compile( name = "common_py_proto", deps = [":common_proto"], ) proto_library( name = "gcs_proto", srcs = ["src/ray/protobuf/gcs.proto"], visibility = ["//java:__subpackages__"], deps = [":common_proto"], ) cc_proto_library( name = "gcs_cc_proto", deps = [":gcs_proto"], ) python_grpc_compile( name = "gcs_py_proto", deps = [":gcs_proto"], ) proto_library( name = "node_manager_proto", srcs = ["src/ray/protobuf/node_manager.proto"], deps = [":common_proto"], ) cc_proto_library( name = "node_manager_cc_proto", deps = [":node_manager_proto"], ) python_grpc_compile( name = "node_manager_py_proto", deps = [":node_manager_proto"], ) proto_library( name = "reporter_proto", srcs = ["src/ray/protobuf/reporter.proto"], deps = [":common_proto"], ) python_grpc_compile( name = "reporter_py_proto", deps = [":reporter_proto"], ) proto_library( name = "gcs_service_proto", srcs = ["src/ray/protobuf/gcs_service.proto"], deps = [ ":common_proto", ":gcs_proto", ], ) cc_proto_library( name = "gcs_service_cc_proto", deps = [":gcs_service_proto"], ) proto_library( name = "object_manager_proto", srcs = ["src/ray/protobuf/object_manager.proto"], ) cc_proto_library( name = "object_manager_cc_proto", deps = [":object_manager_proto"], ) proto_library( name = "core_worker_proto", srcs = ["src/ray/protobuf/core_worker.proto"], deps = [":common_proto"], ) python_grpc_compile( name = "core_worker_py_proto", deps = [":core_worker_proto"], ) cc_proto_library( name = "worker_cc_proto", deps = ["core_worker_proto"], ) proto_library( name = "serialization_proto", srcs = ["src/ray/protobuf/serialization.proto"], ) cc_proto_library( name = "serialization_cc_proto", deps = ["serialization_proto"], ) # === End of protobuf definitions === # === Begin of rpc definitions === # GRPC common lib. cc_library( name = "grpc_common_lib", srcs = glob([ "src/ray/rpc/*.cc", ]), hdrs = glob([ "src/ray/rpc/*.h", ]), copts = COPTS, deps = [ ":ray_common", "@boost//:asio", "@com_github_grpc_grpc//:grpc++", "@com_google_protobuf//:protobuf", ], ) # Node manager gRPC lib. cc_grpc_library( name = "node_manager_cc_grpc", srcs = [":node_manager_proto"], grpc_only = True, deps = [":node_manager_cc_proto"], ) # Node manager server and client. cc_library( name = "node_manager_rpc", hdrs = glob([ "src/ray/rpc/node_manager/*.h", ]), copts = COPTS, deps = [ ":grpc_common_lib", ":node_manager_cc_grpc", ":ray_common", "@boost//:asio", "@com_github_grpc_grpc//:grpc++", ], ) # gcs_service gRPC lib. cc_grpc_library( name = "gcs_service_cc_grpc", srcs = [":gcs_service_proto"], grpc_only = True, deps = [":gcs_service_cc_proto"], ) # gcs rpc server and client. cc_library( name = "gcs_service_rpc", hdrs = glob([ "src/ray/rpc/gcs_server/gcs_rpc_server.h", "src/ray/rpc/gcs_server/gcs_rpc_client.h", ]), copts = COPTS, deps = [ ":gcs_service_cc_grpc", ":grpc_common_lib", ":ray_common", "@boost//:asio", "@com_github_grpc_grpc//:grpc++", ], ) # Object manager gRPC lib. cc_grpc_library( name = "object_manager_cc_grpc", srcs = [":object_manager_proto"], grpc_only = True, deps = [":object_manager_cc_proto"], ) # Object manager rpc server and client. cc_library( name = "object_manager_rpc", hdrs = glob([ "src/ray/rpc/object_manager/*.h", ]), copts = COPTS, deps = [ ":grpc_common_lib", ":object_manager_cc_grpc", ":ray_common", "@boost//:asio", "@com_github_grpc_grpc//:grpc++", ], ) # Worker gRPC lib. cc_grpc_library( name = "worker_cc_grpc", srcs = [":core_worker_proto"], grpc_only = True, deps = [":worker_cc_proto"], ) # worker server and client. cc_library( name = "worker_rpc", hdrs = glob([ "src/ray/rpc/worker/*.h", ]), copts = COPTS, deps = [ ":grpc_common_lib", ":ray_common", ":worker_cc_grpc", "@boost//:asio", "@boost//:thread", "@com_github_grpc_grpc//:grpc++", ], ) # === End of rpc definitions === # === Begin of plasma definitions === # TODO(mehrdadn): (How to) support dynamic linking? PROPAGATED_WINDOWS_DEFINES = ["ARROW_STATIC"] PLASMA_COPTS = COPTS + select({ "@bazel_tools//src/conditions:windows": [ "-D" + "WIN32_REPLACE_FD_APIS", "/FI" + "win32fd.h", ] + ["-D" + define for define in PROPAGATED_WINDOWS_DEFINES], "//conditions:default": [ "-DARROW_USE_GLOG", ], }) PLASMA_LINKOPTS = [] + select({ "@bazel_tools//src/conditions:windows": [ "-DefaultLib:" + "ws2_32.lib", ], "//conditions:default": [ ], }) cc_library( name = "plasma_client", srcs = [ "src/ray/plasma/client.cc", "src/ray/plasma/common.cc", "src/ray/plasma/fling.cc", "src/ray/plasma/io.cc", "src/ray/plasma/malloc.cc", "src/ray/plasma/plasma.cc", "src/ray/plasma/protocol.cc", ], hdrs = [ "src/ray/plasma/client.h", "src/ray/plasma/common.h", "src/ray/plasma/common_generated.h", "src/ray/plasma/compat.h", "src/ray/plasma/external_store.h", "src/ray/plasma/fling.h", "src/ray/plasma/io.h", "src/ray/plasma/malloc.h", "src/ray/plasma/plasma.h", "src/ray/plasma/plasma_generated.h", "src/ray/plasma/protocol.h", ], copts = PLASMA_COPTS, defines = select({ "@bazel_tools//src/conditions:windows": PROPAGATED_WINDOWS_DEFINES, "//conditions:default": [], }), linkopts = PLASMA_LINKOPTS, strip_include_prefix = "src/ray", deps = [ ":common_fbs", ":plasma_fbs", ":platform_shims", "@arrow", "@com_github_google_glog//:glog", ], ) cc_binary( name = "libplasma_java.so", srcs = [ "src/ray/plasma/lib/java/org_apache_arrow_plasma_PlasmaClientJNI.cc", "src/ray/plasma/lib/java/org_apache_arrow_plasma_PlasmaClientJNI.h", ":jni.h", ":jni_md.h", ], copts = PLASMA_COPTS, includes = [ "src/ray", ], linkshared = 1, linkstatic = 1, deps = [":plasma_client"], ) genrule( name = "copy_jni_h", srcs = ["@bazel_tools//tools/jdk:jni_header"], outs = ["jni.h"], cmd = "cp -f $< $@", ) genrule( name = "copy_jni_md_h", srcs = select({ "@bazel_tools//src/conditions:windows": ["@bazel_tools//tools/jdk:jni_md_header-windows"], "@bazel_tools//src/conditions:darwin": ["@bazel_tools//tools/jdk:jni_md_header-darwin"], "//conditions:default": ["@bazel_tools//tools/jdk:jni_md_header-linux"], }), outs = ["jni_md.h"], cmd = "cp -f $< $@", ) genrule( name = "plasma-jni-darwin-compat", srcs = [":libplasma_java.so"], outs = ["libplasma_java.dylib"], cmd = "cp $< $@", output_to_bindir = 1, ) cc_library( name = "ae", srcs = [ "src/ray/plasma/thirdparty/ae/ae.c", ], hdrs = [ "src/ray/plasma/thirdparty/ae/ae.h", "src/ray/plasma/thirdparty/ae/ae_epoll.c", "src/ray/plasma/thirdparty/ae/ae_evport.c", "src/ray/plasma/thirdparty/ae/ae_kqueue.c", "src/ray/plasma/thirdparty/ae/ae_select.c", "src/ray/plasma/thirdparty/ae/config.h", "src/ray/plasma/thirdparty/ae/zmalloc.h", ], copts = PLASMA_COPTS, includes = [ "src/ray/plasma/thirdparty/ae", ], strip_include_prefix = "src/ray", deps = [ ":platform_shims", ], ) cc_library( name = "plasma_store_server_lib", srcs = [ "src/ray/plasma/dlmalloc.cc", "src/ray/plasma/events.cc", "src/ray/plasma/eviction_policy.cc", "src/ray/plasma/external_store.cc", "src/ray/plasma/plasma_allocator.cc", "src/ray/plasma/quota_aware_policy.cc", "src/ray/plasma/store.cc", ], hdrs = [ "src/ray/plasma/events.h", "src/ray/plasma/eviction_policy.h", "src/ray/plasma/external_store.h", "src/ray/plasma/plasma_allocator.h", "src/ray/plasma/quota_aware_policy.h", "src/ray/plasma/store.h", "src/ray/plasma/thirdparty/dlmalloc.c", ], copts = PLASMA_COPTS, linkopts = PLASMA_LINKOPTS, strip_include_prefix = "src/ray", deps = [ ":ae", ":plasma_client", ":platform_shims", "@com_github_google_glog//:glog", ], ) cc_binary( name = "plasma_store_server", srcs = [ "src/ray/plasma/store_exec.cc", ], copts = PLASMA_COPTS, visibility = ["//visibility:public"], deps = [ ":plasma_store_server_lib", ":platform_shims", ], ) FLATC_ARGS = [ "--gen-object-api", "--gen-mutable", "--scoped-enums", ] flatbuffer_cc_library( name = "common_fbs", srcs = ["src/ray/plasma/common.fbs"], flatc_args = FLATC_ARGS, out_prefix = "src/ray/plasma/", ) flatbuffer_cc_library( name = "plasma_fbs", srcs = ["src/ray/plasma/plasma.fbs"], flatc_args = FLATC_ARGS, includes = ["src/ray/plasma/common.fbs"], out_prefix = "src/ray/plasma/", ) # === End of plasma definitions === cc_library( name = "ray_common", srcs = glob( [ "src/ray/common/**/*.cc", ], exclude = [ "src/ray/common/**/*_test.cc", ], ), hdrs = glob( [ "src/ray/common/**/*.h", ], ), copts = COPTS, includes = [ "@boost//:asio", ], visibility = ["//visibility:public"], deps = [ ":common_cc_proto", ":gcs_cc_proto", ":node_manager_fbs", ":plasma_client", ":ray_util", "@boost//:asio", "@com_github_grpc_grpc//:grpc++", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/memory", "@com_google_googletest//:gtest", "@msgpack", ], ) cc_binary( name = "raylet", srcs = ["src/ray/raylet/main.cc"], copts = COPTS, visibility = ["//java:__subpackages__"], deps = [ ":ray_util", ":raylet_lib", "@com_github_gflags_gflags//:gflags", ], ) cc_binary( name = "raylet_monitor", srcs = [ "src/ray/raylet/monitor.cc", "src/ray/raylet/monitor.h", "src/ray/raylet/monitor_main.cc", ], copts = COPTS, deps = [ ":gcs", ":ray_util", ], ) cc_library( name = "gcs_pub_sub_lib", srcs = glob( [ "src/ray/gcs/pubsub/gcs_pub_sub.cc", ], ), hdrs = glob( [ "src/ray/gcs/pubsub/gcs_pub_sub.h", ], ), copts = COPTS, deps = [ ":gcs", ":ray_common", ":redis_client", ], ) cc_test( name = "gcs_pub_sub_test", srcs = ["src/ray/gcs/pubsub/test/gcs_pub_sub_test.cc"], args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], copts = COPTS, data = [ "//:libray_redis_module.so", "//:redis-cli", "//:redis-server", ], deps = [ ":gcs_pub_sub_lib", "@com_google_googletest//:gtest_main", ], ) cc_library( name = "gcs_server_lib", srcs = glob( [ "src/ray/gcs/gcs_server/*.cc", ], exclude = [ "src/ray/gcs/gcs_server/gcs_server_main.cc", "src/ray/gcs/gcs_server/test/*.cc", ], ), hdrs = glob( [ "src/ray/gcs/gcs_server/*.h", ], ), copts = COPTS, deps = [ ":gcs", ":gcs_pub_sub_lib", ":gcs_service_rpc", ":gcs_table_storage_lib", ":node_manager_rpc", ":raylet_lib", ":worker_rpc", ], ) cc_binary( name = "gcs_server", srcs = [ "src/ray/gcs/gcs_server/gcs_server_main.cc", ], copts = COPTS, visibility = ["//java:__subpackages__"], deps = [ ":gcs_server_lib", "@com_github_gflags_gflags//:gflags", ], ) cc_library( name = "stats_lib", srcs = glob( [ "src/ray/stats/*.cc", ], exclude = [ "src/ray/stats/*_test.cc", ], ), hdrs = glob( [ "src/ray/stats/*.h", ], ), copts = COPTS, includes = [ "src", ], linkopts = select({ "@bazel_tools//src/conditions:windows": [ ], "//conditions:default": [ "-lpthread", ], }), deps = [ ":ray_util", "@com_github_jupp0r_prometheus_cpp//pull", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "@com_google_googletest//:gtest", "@io_opencensus_cpp//opencensus/exporters/stats/prometheus:prometheus_exporter", "@io_opencensus_cpp//opencensus/exporters/stats/stdout:stdout_exporter", "@io_opencensus_cpp//opencensus/stats", "@io_opencensus_cpp//opencensus/tags", ], ) cc_library( name = "raylet_lib", srcs = glob( [ "src/ray/raylet/*.cc", ], exclude = [ "src/ray/raylet/mock_gcs_client.cc", "src/ray/raylet/monitor*.cc", "src/ray/raylet/*_test.cc", "src/ray/raylet/main.cc", ], ), hdrs = glob([ "src/ray/raylet/*.h", ]), copts = COPTS, linkopts = select({ "@bazel_tools//src/conditions:windows": [ ], "//conditions:default": [ "-lpthread", ], }), visibility = ["//streaming:__subpackages__"], deps = [ ":common_cc_proto", ":gcs", ":node_manager_fbs", ":node_manager_rpc", ":object_manager", ":plasma_client", ":ray_common", ":ray_util", ":service_based_gcs_client_lib", ":stats_lib", ":worker_rpc", "@boost//:asio", "@com_github_jupp0r_prometheus_cpp//pull", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "@com_google_googletest//:gtest", "@io_opencensus_cpp//opencensus/exporters/stats/prometheus:prometheus_exporter", "@io_opencensus_cpp//opencensus/stats", "@io_opencensus_cpp//opencensus/tags", ], ) cc_library( name = "core_worker_lib", srcs = glob( [ "src/ray/core_worker/*.cc", "src/ray/core_worker/store_provider/*.cc", "src/ray/core_worker/store_provider/memory_store/*.cc", "src/ray/core_worker/transport/*.cc", "src/ray/rpc/worker/*.cc", ], exclude = [ "src/ray/core_worker/*_test.cc", "src/ray/core_worker/mock_worker.cc", ], ), hdrs = glob([ "src/ray/core_worker/*.h", "src/ray/core_worker/store_provider/*.h", "src/ray/core_worker/store_provider/memory_store/*.h", "src/ray/core_worker/transport/*.h", ]), copts = COPTS, visibility = ["//visibility:public"], deps = [ "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", ":worker_cc_proto", ":ray_common", ":ray_util", # TODO(hchen): After `raylet_client` is migrated to gRPC, `core_worker_lib` # should only depend on `raylet_client`, instead of the whole `raylet_lib`. ":raylet_lib", ":worker_rpc", ":gcs", "@boost//:fiber", ], ) cc_library( name = "mock_worker_lib", srcs = ["src/ray/core_worker/test/mock_worker.cc"], hdrs = glob([ "src/ray/core_worker/test/*.h", ]), copts = COPTS, deps = [ ":core_worker_lib", ], ) cc_binary( name = "mock_worker", copts = COPTS, deps = [ ":mock_worker_lib", ], ) cc_test( name = "core_worker_test", srcs = ["src/ray/core_worker/test/core_worker_test.cc"], args = ["$(location //:plasma_store_server) $(location raylet) $(location raylet_monitor) $(location mock_worker) $(location gcs_server) $(location redis-cli) $(location redis-server) $(location libray_redis_module.so)"], copts = COPTS, data = [ "//:gcs_server", "//:libray_redis_module.so", "//:mock_worker", "//:plasma_store_server", "//:raylet", "//:raylet_monitor", "//:redis-cli", "//:redis-server", ], deps = [ ":core_worker_lib", ":gcs", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "direct_actor_transport_test", srcs = ["src/ray/core_worker/test/direct_actor_transport_test.cc"], copts = COPTS, deps = [ ":core_worker_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "direct_task_transport_test", srcs = ["src/ray/core_worker/test/direct_task_transport_test.cc"], copts = COPTS, deps = [ ":core_worker_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "reference_count_test", srcs = ["src/ray/core_worker/reference_count_test.cc"], copts = COPTS, deps = [ ":core_worker_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "object_recovery_manager_test", srcs = ["src/ray/core_worker/test/object_recovery_manager_test.cc"], copts = COPTS, deps = [ ":core_worker_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "scheduling_queue_test", srcs = ["src/ray/core_worker/test/scheduling_queue_test.cc"], copts = COPTS, deps = [ ":core_worker_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "task_manager_test", srcs = ["src/ray/core_worker/test/task_manager_test.cc"], copts = COPTS, deps = [ ":core_worker_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "scheduling_test", srcs = ["src/ray/common/scheduling/scheduling_test.cc"], copts = COPTS, deps = [ ":ray_common", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "lineage_cache_test", srcs = ["src/ray/raylet/lineage_cache_test.cc"], copts = COPTS, deps = [ ":node_manager_fbs", ":raylet_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "reconstruction_policy_test", srcs = ["src/ray/raylet/reconstruction_policy_test.cc"], copts = COPTS, deps = [ ":node_manager_fbs", ":object_manager", ":raylet_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "worker_pool_test", srcs = ["src/ray/raylet/worker_pool_test.cc"], copts = COPTS, deps = [ ":raylet_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "id_test", srcs = ["src/ray/common/id_test.cc"], copts = COPTS, deps = [ "ray_common", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "logging_test", srcs = ["src/ray/util/logging_test.cc"], args = ["--gtest_filter=PrintLogTest*"], copts = COPTS, deps = [ ":ray_util", "@boost//:asio", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "util_test", srcs = ["src/ray/util/util_test.cc"], copts = COPTS, deps = [ ":ray_util", "@boost//:asio", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "sample_test", srcs = ["src/ray/util/sample_test.cc"], copts = COPTS, deps = [ ":ray_common", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "task_dependency_manager_test", srcs = ["src/ray/raylet/task_dependency_manager_test.cc"], copts = COPTS, deps = [ ":raylet_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "client_connection_test", srcs = ["src/ray/raylet/client_connection_test.cc"], copts = COPTS, deps = [ ":raylet_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "signal_test", srcs = ["src/ray/util/signal_test.cc"], copts = COPTS, deps = [ ":raylet_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "sequencer_test", srcs = ["src/ray/util/sequencer_test.cc"], copts = COPTS, deps = [ ":ray_util", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "stats_test", srcs = ["src/ray/stats/stats_test.cc"], copts = COPTS, deps = [ ":stats_lib", "@com_google_googletest//:gtest_main", ], ) cc_library( name = "gcs_test_util_lib", hdrs = [ "src/ray/gcs/test/accessor_test_base.h", "src/ray/gcs/test/gcs_test_util.h", ], copts = COPTS, deps = [ ":gcs", ":gcs_service_rpc", ], ) cc_test( name = "gcs_server_rpc_test", srcs = [ "src/ray/gcs/gcs_server/test/gcs_server_rpc_test.cc", ], args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], copts = COPTS, data = [ "//:libray_redis_module.so", "//:redis-cli", "//:redis-server", ], deps = [ ":gcs_server_lib", ":gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "gcs_node_manager_test", srcs = [ "src/ray/gcs/gcs_server/test/gcs_node_manager_test.cc", "src/ray/gcs/gcs_server/test/gcs_server_test_util.h", ], copts = COPTS, deps = [ ":gcs_server_lib", ":gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "gcs_actor_scheduler_test", srcs = [ "src/ray/gcs/gcs_server/test/gcs_actor_scheduler_test.cc", "src/ray/gcs/gcs_server/test/gcs_server_test_util.h", ], copts = COPTS, deps = [ ":gcs_server_lib", ":gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "gcs_actor_manager_test", srcs = [ "src/ray/gcs/gcs_server/test/gcs_actor_manager_test.cc", "src/ray/gcs/gcs_server/test/gcs_server_test_util.h", ], copts = COPTS, deps = [ ":gcs_server_lib", ":gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) cc_library( name = "gcs_table_storage_lib", srcs = glob( [ "src/ray/gcs/gcs_server/gcs_table_storage.cc", ], ), hdrs = glob( [ "src/ray/gcs/gcs_server/gcs_table_storage.h", ], ), copts = COPTS, deps = [ ":gcs", ":gcs_in_memory_store_client", ":ray_common", ":redis_store_client", ], ) cc_library( name = "gcs_table_storage_test_lib", hdrs = [ "src/ray/gcs/gcs_server/test/gcs_table_storage_test_base.h", ], copts = COPTS, deps = [ "redis_store_client", ], ) cc_test( name = "redis_gcs_table_storage_test", srcs = [ "src/ray/gcs/gcs_server/test/redis_gcs_table_storage_test.cc", ], args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], copts = COPTS, data = [ "//:libray_redis_module.so", "//:redis-cli", "//:redis-server", ], deps = [ ":gcs_table_storage_lib", ":gcs_table_storage_test_lib", ":gcs_test_util_lib", ":store_client_test_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "in_memory_gcs_table_storage_test", srcs = [ "src/ray/gcs/gcs_server/test/in_memory_gcs_table_storage_test.cc", ], copts = COPTS, deps = [ ":gcs_table_storage_lib", ":gcs_table_storage_test_lib", ":gcs_test_util_lib", ":store_client_test_lib", "@com_google_googletest//:gtest_main", ], ) cc_library( name = "service_based_gcs_client_lib", srcs = glob( [ "src/ray/gcs/gcs_client/service_based_*.cc", ], ), hdrs = glob( [ "src/ray/gcs/gcs_client/service_based_*.h", ], ), copts = COPTS, deps = [ ":gcs", ":gcs_pub_sub_lib", ":gcs_service_rpc", ":redis_store_client", ], ) cc_library( name = "global_state_accessor_lib", srcs = glob( [ "src/ray/gcs/gcs_client/global_state_accessor.cc", ], ), hdrs = glob( [ "src/ray/gcs/gcs_client/global_state_accessor.h", ], ), copts = COPTS, deps = [ ":service_based_gcs_client_lib", ], ) cc_test( name = "global_state_accessor_test", srcs = [ "src/ray/gcs/gcs_client/test/global_state_accessor_test.cc", ], args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], copts = COPTS, data = [ "//:libray_redis_module.so", "//:redis-cli", "//:redis-server", ], deps = [ ":gcs_server_lib", ":gcs_test_util_lib", ":global_state_accessor_lib", ":service_based_gcs_client_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "gcs_server_test", srcs = [ "src/ray/gcs/gcs_client/test/service_based_gcs_client_test.cc", ], args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], copts = COPTS, data = [ "//:libray_redis_module.so", "//:redis-cli", "//:redis-server", ], # TODO(swang): Enable again once pubsub client supports GCS server restart. tags = ["manual"], deps = [ ":gcs_server_lib", ":gcs_test_util_lib", ":service_based_gcs_client_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "gcs_object_manager_test", srcs = [ "src/ray/gcs/gcs_server/test/gcs_object_manager_test.cc", "src/ray/gcs/gcs_server/test/gcs_server_test_util.h", ], copts = COPTS, deps = [ ":gcs_server_lib", ":gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) cc_library( name = "object_manager", srcs = glob([ "src/ray/object_manager/*.cc", ]), hdrs = glob([ "src/ray/object_manager/*.h", ]), copts = COPTS, includes = [ "src", ], deps = [ ":gcs", ":object_manager_fbs", ":object_manager_rpc", ":plasma_client", ":ray_common", ":ray_util", "@boost//:asio", ], ) cc_binary( name = "object_manager_test", testonly = 1, srcs = ["src/ray/object_manager/test/object_manager_test.cc"], copts = COPTS, deps = [ ":object_manager", "@com_google_googletest//:gtest_main", ], ) cc_binary( name = "object_manager_stress_test", testonly = 1, srcs = ["src/ray/object_manager/test/object_manager_stress_test.cc"], copts = COPTS, deps = [ ":object_manager", "@com_google_googletest//:gtest_main", ], ) cc_library( name = "platform_shims", srcs = [] + select({ "@bazel_tools//src/conditions:windows": glob([ "src/shims/windows/**/*.c", "src/shims/windows/**/*.cc", "src/shims/windows/**/*.h", ]), "//conditions:default": [], }), hdrs = [] + select({ "@bazel_tools//src/conditions:windows": glob([ "src/shims/windows/**/*.h", ]), "//conditions:default": [], }), copts = COPTS, includes = [] + select({ "@bazel_tools//src/conditions:windows": [ "src/shims/windows", ], "//conditions:default": [], }), visibility = ["//visibility:public"], ) cc_library( name = "ray_util", srcs = glob( [ "src/ray/util/*.cc", ], exclude = [ "src/ray/util/*_test.cc", ], ), hdrs = glob([ "src/ray/util/*.h", ]), copts = COPTS, includes = [ "src", ], visibility = ["//visibility:public"], deps = [ ":plasma_client", ":sha256", "@boost//:asio", "@com_github_google_glog//:glog", "@com_google_absl//absl/synchronization", "@com_google_absl//absl/time", "@com_google_googletest//:gtest_main", ], ) filegroup( name = "iwyu_sh", srcs = ["ci/travis/iwyu.sh"], ) filegroup( name = "extra_actions_base_proto", srcs = [ # TODO: Replace our file with the built-in copy once this issue is resolved: # https://github.com/bazelbuild/bazel/issues/8738 "thirdparty/protobuf/extra_actions_base.proto", #"@bazel_tools//src/main/protobuf:extra_actions_base.proto", ], ) action_listener( name = "iwyu_cpp", extra_actions = [":iwyu_action"], mnemonics = ["CppCompile"], ) extra_action( name = "iwyu_action", cmd = "$(location :iwyu_sh) $(location @com_google_protobuf//:protoc) $(location :extra_actions_base_proto) --extra_action_file=$(EXTRA_ACTION_FILE)", tools = [ ":extra_actions_base_proto", ":iwyu_sh", "@com_google_protobuf//:protoc", ], ) cc_library( name = "sha256", srcs = [ "src/ray/thirdparty/sha256.c", ], hdrs = [ "src/ray/thirdparty/sha256.h", ], copts = COPTS, includes = ["src/ray/thirdparty"], ) alias( name = "hiredis", actual = "@com_github_antirez_redis//:hiredis", ) cc_library( name = "redis_client", srcs = [ "src/ray/gcs/asio.cc", "src/ray/gcs/redis_async_context.cc", "src/ray/gcs/redis_client.cc", "src/ray/gcs/redis_context.cc", ], hdrs = [ "src/ray/gcs/asio.h", "src/ray/gcs/redis_async_context.h", "src/ray/gcs/redis_client.h", "src/ray/gcs/redis_context.h", ], copts = COPTS, deps = [ ":hiredis", ":ray_common", ":ray_util", ":stats_lib", "@boost//:asio", ], ) cc_library( name = "redis_store_client", srcs = [ "src/ray/gcs/store_client/redis_store_client.cc", ], hdrs = [ "src/ray/gcs/callback.h", "src/ray/gcs/store_client/redis_store_client.h", "src/ray/gcs/store_client/store_client.h", ], copts = COPTS, deps = [ "redis_client", ], ) cc_library( name = "gcs_in_memory_store_client", srcs = [ "src/ray/gcs/store_client/in_memory_store_client.cc", ], hdrs = [ "src/ray/gcs/callback.h", "src/ray/gcs/store_client/in_memory_store_client.h", "src/ray/gcs/store_client/store_client.h", ], copts = COPTS, deps = [ ":ray_common", ":ray_util", ], ) cc_library( name = "store_client_test_lib", hdrs = [ "src/ray/gcs/store_client/test/store_client_test_base.h", ], copts = COPTS, deps = [ "redis_store_client", ], ) cc_test( name = "redis_store_client_test", srcs = ["src/ray/gcs/store_client/test/redis_store_client_test.cc"], args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], copts = COPTS, data = [ "//:libray_redis_module.so", "//:redis-cli", "//:redis-server", ], deps = [ ":redis_store_client", ":store_client_test_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "in_memory_store_client_test", srcs = ["src/ray/gcs/store_client/test/in_memory_store_client_test.cc"], copts = COPTS, deps = [ ":gcs_in_memory_store_client", ":store_client_test_lib", "@com_google_googletest//:gtest_main", ], ) cc_library( name = "gcs", srcs = glob( [ "src/ray/gcs/*.cc", ], exclude = [ "src/ray/gcs/*_test.cc", ], ), hdrs = glob([ "src/ray/gcs/*.h", ]), copts = COPTS, deps = [ ":gcs_cc_proto", ":hiredis", ":node_manager_fbs", ":node_manager_rpc", ":ray_common", ":ray_util", ":stats_lib", "@boost//:asio", ], ) # TODO(micafan) Support test group in future. Use test group we can run all gcs test once. cc_test( name = "redis_gcs_client_test", srcs = ["src/ray/gcs/test/redis_gcs_client_test.cc"], args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], copts = COPTS, data = [ "//:libray_redis_module.so", "//:redis-cli", "//:redis-server", ], deps = [ ":gcs", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "redis_actor_info_accessor_test", srcs = ["src/ray/gcs/test/redis_actor_info_accessor_test.cc"], args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], copts = COPTS, data = [ "//:libray_redis_module.so", "//:redis-cli", "//:redis-server", ], deps = [ ":gcs", ":gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "redis_object_info_accessor_test", srcs = ["src/ray/gcs/test/redis_object_info_accessor_test.cc"], args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], copts = COPTS, data = [ "//:libray_redis_module.so", "//:redis-cli", "//:redis-server", ], deps = [ ":gcs", ":gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "subscription_executor_test", srcs = ["src/ray/gcs/test/subscription_executor_test.cc"], args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], copts = COPTS, data = [ "//:libray_redis_module.so", "//:redis-cli", "//:redis-server", ], deps = [ ":gcs", ":gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "redis_job_info_accessor_test", srcs = ["src/ray/gcs/test/redis_job_info_accessor_test.cc"], args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], copts = COPTS, data = [ "//:libray_redis_module.so", "//:redis-cli", "//:redis-server", ], deps = [ ":gcs", ":gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "redis_node_info_accessor_test", srcs = ["src/ray/gcs/test/redis_node_info_accessor_test.cc"], args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], copts = COPTS, data = [ "//:libray_redis_module.so", "//:redis-cli", "//:redis-server", ], deps = [ ":gcs", ":gcs_test_util_lib", "@com_google_googletest//:gtest_main", ], ) cc_test( name = "asio_test", srcs = ["src/ray/gcs/test/asio_test.cc"], args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], copts = COPTS, data = [ "//:libray_redis_module.so", "//:redis-cli", "//:redis-server", ], deps = [ ":gcs", ":ray_util", "@com_google_googletest//:gtest_main", ], ) flatbuffer_cc_library( name = "node_manager_fbs", srcs = ["src/ray/raylet/format/node_manager.fbs"], flatc_args = FLATC_ARGS, out_prefix = "src/ray/raylet/format/", ) flatbuffer_cc_library( name = "object_manager_fbs", srcs = ["src/ray/object_manager/format/object_manager.fbs"], flatc_args = FLATC_ARGS, out_prefix = "src/ray/object_manager/format/", ) pyx_library( name = "_raylet", srcs = glob([ "python/ray/__init__.py", "python/ray/_raylet.pxd", "python/ray/_raylet.pyx", "python/ray/includes/*.pxd", "python/ray/includes/*.pxi", ]), # Export ray ABI symbols, which can then be used by _streaming.so. # We need to dlopen this lib with RTLD_GLOBAL to use ABI in this # shared lib, see python/ray/__init__.py. cc_kwargs = dict( srcs = PYX_SRCS, copts = COPTS + PYX_COPTS, # see https://github.com/tensorflow/tensorflow/blob/r2.1/tensorflow/lite/BUILD#L444 linkopts = select({ "@bazel_tools//src/conditions:darwin": [ "-Wl,-exported_symbols_list,$(location //:src/ray/ray_exported_symbols.lds)", ], "@bazel_tools//src/conditions:windows": [ ], "//conditions:default": [ "-Wl,--version-script,$(location //:src/ray/ray_version_script.lds)", ], }), linkstatic = 1, ), deps = [ "//:core_worker_lib", "//:global_state_accessor_lib", "//:ray_util", "//:raylet_lib", "//:serialization_cc_proto", "//:src/ray/ray_exported_symbols.lds", "//:src/ray/ray_version_script.lds", ], ) pyx_library( name = "_streaming", srcs = glob([ "python/ray/streaming/_streaming.pyx", "python/ray/__init__.py", "python/ray/_raylet.pxd", "python/ray/includes/*.pxd", "python/ray/includes/*.pxi", "python/ray/streaming/__init__.pxd", "python/ray/streaming/includes/*.pxd", "python/ray/streaming/includes/*.pxi", ]), cc_kwargs = dict( srcs = PYX_SRCS, copts = COPTS + PYX_COPTS, ), deps = [ "//streaming:streaming_lib", ], ) cc_binary( name = "libcore_worker_library_java.so", srcs = glob([ "src/ray/core_worker/lib/java/*.h", "src/ray/core_worker/lib/java/*.cc", ]) + [ "@bazel_tools//tools/jdk:jni_header", ] + select({ "@bazel_tools//src/conditions:windows": ["@bazel_tools//tools/jdk:jni_md_header-windows"], "@bazel_tools//src/conditions:darwin": ["@bazel_tools//tools/jdk:jni_md_header-darwin"], "//conditions:default": ["@bazel_tools//tools/jdk:jni_md_header-linux"], }) + [ ":jni.h", ":jni_md.h", ], copts = COPTS, includes = [ "src", "external/bazel_tools/tools/jdk/include", ] + select({ "@bazel_tools//src/conditions:windows": ["external/bazel_tools/tools/jdk/include/windows"], "@bazel_tools//src/conditions:darwin": ["external/bazel_tools/tools/jdk/include/darwin"], "//conditions:default": ["external/bazel_tools/tools/jdk/include/linux"], }), # Export ray ABI symbols, which can then be used by libstreaming_java.so. see `//:_raylet` linkopts = select({ "@bazel_tools//src/conditions:darwin": [ "-Wl,-exported_symbols_list,$(location //:src/ray/ray_exported_symbols.lds)", ], "@bazel_tools//src/conditions:windows": [ ], "//conditions:default": [ "-Wl,--version-script,$(location //:src/ray/ray_version_script.lds)", ], }), linkshared = 1, linkstatic = 1, deps = [ "//:core_worker_lib", "//:global_state_accessor_lib", "//:src/ray/ray_exported_symbols.lds", "//:src/ray/ray_version_script.lds", "//streaming:jni", ], ) genrule( name = "core_worker-jni-darwin-compat", srcs = [":libcore_worker_library_java.so"], outs = ["libcore_worker_library_java.dylib"], cmd = "cp $< $@", output_to_bindir = 1, ) filegroup( name = "core_worker_library_java", srcs = select({ "@bazel_tools//src/conditions:darwin": [":libcore_worker_library_java.dylib"], "//conditions:default": [":libcore_worker_library_java.so"], }), visibility = ["//java:__subpackages__"], ) filegroup( name = "python_sources", srcs = glob([ "python/ray/*.py", "python/ray/autoscaler/*.py", "python/ray/autoscaler/aws/example-full.yaml", "python/ray/autoscaler/azure/example-full.yaml", "python/ray/autoscaler/gcp/example-full.yaml", "python/ray/autoscaler/local/example-full.yaml", "python/ray/cloudpickle/*.py", "python/ray/core/__init__.py", "python/ray/core/generated/__init__.py", "python/ray/core/generated/ray/__init__.py", "python/ray/core/generated/ray/protocol/__init__.py", "python/ray/dashboard/*.py", "python/ray/dashboard/metrics_exporter/*.py", "python/ray/experimental/*.py", "python/ray/util/*.py", "python/ray/internal/*.py", "python/ray/projects/*.py", "python/ray/projects/schema.json", "python/ray/projects/templates/cluster_template.yaml", "python/ray/projects/templates/project_template.yaml", "python/ray/projects/templates/requirements.txt", "python/ray/workers/default_worker.py", ]), ) genrule( name = "redis", srcs = [ ] + select({ "@bazel_tools//src/conditions:windows": [ "@com_github_tporadowski_redis_bin//file", ], "//conditions:default": [ "@com_github_antirez_redis//:file", # This is necessary so we can access the root of the directory "@com_github_antirez_redis//:files", # This is necessary to ensure entire directory tree is accessible ], }), outs = [ "redis-server", "redis-cli", ], cmd = select({ "@bazel_tools//src/conditions:windows": """ unzip -q -o -- $(location @com_github_tporadowski_redis_bin//file) redis-server.exe redis-cli.exe && mv -f -- redis-server.exe $(location redis-server) && mv -f -- redis-cli.exe $(location redis-cli) """, "//conditions:default": """ tmpdir="redis.tmp" && path=$(location @com_github_antirez_redis//:file) && cp -p -L -R -- "$${path%/*}" "$${tmpdir}" && chmod +x "$${tmpdir}"/deps/jemalloc/configure && parallel="$$(getconf _NPROCESSORS_ONLN || echo 1)" make -s -C "$${tmpdir}" -j"$${parallel}" V=0 CFLAGS="$${CFLAGS-} -DLUA_USE_MKSTEMP -Wno-pragmas -Wno-empty-body" && mv "$${tmpdir}"/src/redis-server $(location redis-server) && chmod +x $(location redis-server) && mv "$${tmpdir}"/src/redis-cli $(location redis-cli) && chmod +x $(location redis-cli) && rm -r -f -- "$${tmpdir}" """, }), visibility = ["//java:__subpackages__"], ) cc_binary( name = "libray_redis_module.so", srcs = [ "src/ray/gcs/redis_module/ray_redis_module.cc", "src/ray/gcs/redis_module/redis_string.h", "src/ray/gcs/redis_module/redismodule.h", ], copts = COPTS, linkshared = 1, linkstatic = 1, visibility = ["//java:__subpackages__"], deps = [ ":gcs_cc_proto", ":ray_common", ], ) filegroup( name = "all_py_proto", srcs = [ "common_py_proto", "core_worker_py_proto", "gcs_py_proto", "node_manager_py_proto", "reporter_py_proto", ], ) # This is a dummy test dependency that causes the python tests to be # re-run if any of these files changes. py_library( name = "ray_lib", srcs = glob( ["python/ray/**/*.py"], exclude = ["python/ray/tests/*.py"], ), visibility = ["__subpackages__"], ) genrule( name = "ray_pkg", srcs = [ "python/ray/_raylet.so", "python/ray/streaming/_streaming.so", "//:python_sources", "//:all_py_proto", "//:redis-server", "//:redis-cli", "//:libray_redis_module.so", "//:raylet", "//:raylet_monitor", "//:gcs_server", "//:plasma_store_server", "//streaming:copy_streaming_py_proto", ], outs = ["ray_pkg.out"], cmd = """ WORK_DIR="$$(pwd)" && RAYDIR="$$WORK_DIR/python/ray" RAYLET="$(location python/ray/_raylet.so)" RAYLET_TARGET_FILENAME="$${RAYLET##*/}" && if [ "$${OSTYPE-}" = "msys" ]; then # If Windows, we need to do things differently (e.g. use .pyd instead of .so) RAYLET_TARGET_FILENAME="$${RAYLET_TARGET_FILENAME%.*}" && RAYLET_TARGET_FILENAME="$${RAYLET_TARGET_FILENAME}.pyd" && if [ -f "$${RAYLET%.*}.pdb" ]; then cp -f "$${RAYLET%.*}.pdb" "$${RAYDIR}"; fi; fi && cp -f "$${RAYLET}" "$${RAYDIR}/$${RAYLET_TARGET_FILENAME}" && cp -f $(location python/ray/streaming/_streaming.so) "$$WORK_DIR/python/ray/streaming" && mkdir -p "$$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/" && cp -f $(location //:redis-server) "$$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/" && cp -f $(location //:redis-cli) "$$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/" && mkdir -p "$$WORK_DIR/python/ray/core/src/ray/gcs/redis_module/" && cp -f $(locations //:libray_redis_module.so) "$$WORK_DIR/python/ray/core/src/ray/gcs/redis_module/" && cp -f $(location //:raylet_monitor) "$$WORK_DIR/python/ray/core/src/ray/raylet/" && cp -f $(location //:plasma_store_server) "$$WORK_DIR/python/ray/core/src/plasma/" && cp -f $(location //:raylet) "$$WORK_DIR/python/ray/core/src/ray/raylet/" && cp -f $(location //:gcs_server) "$$WORK_DIR/python/ray/core/src/ray/gcs/" && mkdir -p "$$WORK_DIR/python/ray/core/generated/ray/protocol/" && for f in $(locations //:all_py_proto); do cp -f "$$f" "$$WORK_DIR/python/ray/core/generated/"; done && # NOTE(hchen): Protobuf doesn't allow specifying Python package name. So we use this `sed` # command to change the import path in the generated file. sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/gcs_pb2.py" && sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/common_pb2.py" && sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/node_manager_pb2.py" && sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/node_manager_pb2_grpc.py" && sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/reporter_pb2.py" && sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/reporter_pb2_grpc.py" && sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/core_worker_pb2.py" && sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/core_worker_pb2_grpc.py" && echo "$$WORK_DIR" > $@ """, local = 1, )