ray/BUILD.bazel

2677 lines
63 KiB
Text
Raw Normal View History

2019-01-20 12:16:47 -08:00
# Bazel build
# C/C++ documentation: https://docs.bazel.build/versions/master/be/c-cpp.html
load("@rules_proto//proto:defs.bzl", "proto_library")
2020-08-07 16:34:29 -07:00
load("@rules_python//python:defs.bzl", "py_library")
load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library", "cc_test")
load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library")
2019-11-08 15:58:28 -08:00
load("@com_github_grpc_grpc//bazel:cython_library.bzl", "pyx_library")
2019-01-20 12:16:47 -08:00
load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library")
load("//bazel:ray.bzl", "COPTS", "PYX_COPTS", "PYX_SRCS", "copy_to_workspace")
2019-01-27 18:32:04 -08:00
package(
default_visibility = ["//visibility:public"],
)
config_setting(
name = "msvc-cl",
flag_values = {"@bazel_tools//tools/cpp:compiler": "msvc-cl"},
)
config_setting(
name = "clang-cl",
flag_values = {"@bazel_tools//tools/cpp:compiler": "clang-cl"},
)
config_setting(
name = "opt",
values = {"compilation_mode": "opt"},
)
# === Begin of rpc definitions ===
# GRPC common lib.
cc_library(
name = "grpc_common_lib",
srcs = glob([
"src/ray/rpc/*.cc",
]),
hdrs = glob([
"src/ray/rpc/*.h",
"src/ray/raylet_client/*.h",
]),
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
deps = [
":ray_common",
":stats_metric",
"@boost//:asio",
"@com_github_grpc_grpc//:grpc++",
"@com_google_protobuf//:protobuf",
],
)
# Node manager gRPC lib.
cc_grpc_library(
name = "node_manager_cc_grpc",
srcs = ["//src/ray/protobuf:node_manager_proto"],
grpc_only = True,
deps = ["//src/ray/protobuf:node_manager_cc_proto"],
)
# Node manager server and client.
cc_library(
name = "node_manager_rpc",
srcs = glob([
"src/ray/rpc/node_manager/*.cc",
]),
hdrs = glob([
"src/ray/rpc/node_manager/*.h",
]),
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
deps = [
":grpc_common_lib",
":node_manager_cc_grpc",
":ray_common",
"@boost//:asio",
"@com_github_grpc_grpc//:grpc++",
],
)
# gcs_service gRPC lib.
cc_grpc_library(
name = "gcs_service_cc_grpc",
srcs = ["//src/ray/protobuf:gcs_service_proto"],
grpc_only = True,
deps = ["//src/ray/protobuf:gcs_service_cc_proto"],
)
# gcs_service gRPC lib.
cc_grpc_library(
name = "test_service_cc_grpc",
srcs = ["//src/ray/protobuf:test_service_proto"],
grpc_only = True,
deps = ["//src/ray/protobuf:test_service_cc_proto"],
)
# gcs rpc server and client.
cc_library(
name = "gcs_service_rpc",
hdrs = [
"src/ray/rpc/gcs_server/gcs_rpc_client.h",
"src/ray/rpc/gcs_server/gcs_rpc_server.h",
],
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
deps = [
":gcs_service_cc_grpc",
":grpc_common_lib",
":pubsub_lib",
":ray_common",
"@boost//:asio",
"@com_github_grpc_grpc//:grpc++",
],
)
# Object manager gRPC lib.
cc_grpc_library(
name = "object_manager_cc_grpc",
srcs = ["//src/ray/protobuf:object_manager_proto"],
grpc_only = True,
deps = ["//src/ray/protobuf:object_manager_cc_proto"],
)
# Object manager rpc server and client.
cc_library(
name = "object_manager_rpc",
hdrs = glob([
"src/ray/rpc/object_manager/*.h",
]),
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
deps = [
":grpc_common_lib",
":object_manager_cc_grpc",
":ray_common",
"@boost//:asio",
"@com_github_grpc_grpc//:grpc++",
],
)
# Worker gRPC lib.
cc_grpc_library(
name = "worker_cc_grpc",
srcs = ["//src/ray/protobuf:core_worker_proto"],
grpc_only = True,
deps = ["//src/ray/protobuf:worker_cc_proto"],
)
2019-07-30 17:47:17 +08:00
# worker server and client.
cc_library(
name = "worker_rpc",
2020-08-07 16:34:29 -07:00
srcs = glob([
"src/ray/rpc/worker/*.cc",
]),
hdrs = glob([
"src/ray/rpc/worker/*.h",
]),
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
deps = [
":grpc_common_lib",
":pubsub_lib",
":ray_common",
":worker_cc_grpc",
"@boost//:asio",
"@boost//:thread",
"@com_github_grpc_grpc//:grpc++",
],
)
2020-07-14 09:49:16 -07:00
# Metrics Agent gRPC lib.
cc_grpc_library(
name = "reporter_cc_grpc",
srcs = ["//src/ray/protobuf:reporter_proto"],
2020-07-14 09:49:16 -07:00
grpc_only = True,
deps = ["//src/ray/protobuf:reporter_cc_proto"],
2020-07-14 09:49:16 -07:00
)
# Metrics Agent client.
cc_library(
name = "reporter_rpc",
hdrs = glob([
"src/ray/rpc/metrics_agent_client.h",
]),
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
2020-07-14 09:49:16 -07:00
deps = [
":grpc_common_lib",
":ray_common",
":reporter_cc_grpc",
"@boost//:asio",
"@boost//:thread",
"@com_github_grpc_grpc//:grpc++",
],
)
# Agent manager.
cc_grpc_library(
name = "agent_manager_cc_grpc",
srcs = ["//src/ray/protobuf:agent_manager_proto"],
grpc_only = True,
deps = ["//src/ray/protobuf:agent_manager_cc_proto"],
)
cc_library(
name = "agent_manager_rpc",
hdrs = glob([
"src/ray/rpc/agent_manager/*.h",
]),
copts = COPTS,
strip_include_prefix = "src",
deps = [
":agent_manager_cc_grpc",
":grpc_common_lib",
":ray_common",
"@boost//:asio",
"@com_github_grpc_grpc//:grpc++",
],
)
# runtime env.
cc_grpc_library(
name = "runtime_env_cc_grpc",
srcs = ["//src/ray/protobuf:runtime_env_agent_proto"],
grpc_only = True,
deps = [
"//src/ray/protobuf:agent_manager_cc_proto",
"//src/ray/protobuf:common_cc_proto",
"//src/ray/protobuf:runtime_env_agent_cc_proto",
],
)
cc_library(
name = "runtime_env_rpc",
hdrs = glob([
"src/ray/rpc/runtime_env/*.h",
]),
copts = COPTS,
strip_include_prefix = "src",
deps = [
":grpc_common_lib",
":ray_common",
":runtime_env_cc_grpc",
"@boost//:asio",
"@com_github_grpc_grpc//:grpc++",
],
)
# pubsub.
cc_grpc_library(
name = "pubsub_cc_grpc",
srcs = ["//src/ray/protobuf:pubsub_proto"],
grpc_only = True,
deps = [
"//src/ray/protobuf:common_cc_proto",
"//src/ray/protobuf:gcs_cc_proto",
"//src/ray/protobuf:pubsub_cc_proto",
],
)
cc_library(
name = "pubsub_rpc",
copts = COPTS,
deps = [
"pubsub_cc_grpc",
":grpc_common_lib",
":ray_common",
],
)
# === End of rpc definitions ===
2020-06-03 17:44:34 -07:00
# === Begin of plasma definitions ===
# TODO(mehrdadn): (How to) support dynamic linking?
PROPAGATED_WINDOWS_DEFINES = ["ARROW_STATIC"]
PLASMA_COPTS = COPTS + select({
"@bazel_tools//src/conditions:windows": [
] + ["-D" + define for define in PROPAGATED_WINDOWS_DEFINES],
"//conditions:default": [
"-DARROW_USE_GLOG",
],
})
PLASMA_LINKOPTS = [] + select({
"@bazel_tools//src/conditions:windows": [
"-DefaultLib:" + "ws2_32.lib",
],
"//conditions:default": [
],
})
cc_library(
name = "plasma_client",
srcs = [
"src/ray/object_manager/plasma/client.cc",
"src/ray/object_manager/plasma/connection.cc",
"src/ray/object_manager/plasma/malloc.cc",
"src/ray/object_manager/plasma/plasma.cc",
"src/ray/object_manager/plasma/protocol.cc",
"src/ray/object_manager/plasma/shared_memory.cc",
] + select({
"@bazel_tools//src/conditions:windows": [
],
"//conditions:default": [
"src/ray/object_manager/plasma/fling.cc",
],
}),
2020-06-03 17:44:34 -07:00
hdrs = [
"src/ray/object_manager/common.h",
"src/ray/object_manager/plasma/client.h",
"src/ray/object_manager/plasma/common.h",
"src/ray/object_manager/plasma/compat.h",
"src/ray/object_manager/plasma/connection.h",
"src/ray/object_manager/plasma/malloc.h",
"src/ray/object_manager/plasma/plasma.h",
"src/ray/object_manager/plasma/plasma_generated.h",
"src/ray/object_manager/plasma/protocol.h",
"src/ray/object_manager/plasma/shared_memory.h",
] + select({
"@bazel_tools//src/conditions:windows": [
],
"//conditions:default": [
"src/ray/object_manager/plasma/fling.h",
],
}),
2020-06-03 17:44:34 -07:00
copts = PLASMA_COPTS,
defines = select({
"@bazel_tools//src/conditions:windows": PROPAGATED_WINDOWS_DEFINES,
"//conditions:default": [],
}),
linkopts = PLASMA_LINKOPTS,
strip_include_prefix = "src",
2020-06-03 17:44:34 -07:00
deps = [
":plasma_fbs",
":ray_common",
":ray_util",
"//src/ray/protobuf:common_cc_proto",
"@msgpack",
2020-06-03 17:44:34 -07:00
],
)
cc_library(
name = "plasma_store_server_lib",
2020-06-03 17:44:34 -07:00
srcs = [
"src/ray/object_manager/plasma/create_request_queue.cc",
"src/ray/object_manager/plasma/dlmalloc.cc",
"src/ray/object_manager/plasma/eviction_policy.cc",
"src/ray/object_manager/plasma/get_request_queue.cc",
"src/ray/object_manager/plasma/object_lifecycle_manager.cc",
"src/ray/object_manager/plasma/object_store.cc",
"src/ray/object_manager/plasma/plasma_allocator.cc",
"src/ray/object_manager/plasma/stats_collector.cc",
"src/ray/object_manager/plasma/store.cc",
"src/ray/object_manager/plasma/store_runner.cc",
2020-06-03 17:44:34 -07:00
],
hdrs = [
"src/ray/object_manager/common.h",
"src/ray/object_manager/plasma/allocator.h",
"src/ray/object_manager/plasma/create_request_queue.h",
"src/ray/object_manager/plasma/eviction_policy.h",
"src/ray/object_manager/plasma/get_request_queue.h",
"src/ray/object_manager/plasma/object_lifecycle_manager.h",
"src/ray/object_manager/plasma/object_store.h",
"src/ray/object_manager/plasma/plasma_allocator.h",
"src/ray/object_manager/plasma/stats_collector.h",
"src/ray/object_manager/plasma/store.h",
"src/ray/object_manager/plasma/store_runner.h",
"src/ray/thirdparty/dlmalloc.c",
2020-06-03 17:44:34 -07:00
],
copts = PLASMA_COPTS,
linkopts = PLASMA_LINKOPTS,
strip_include_prefix = "src",
2020-06-03 17:44:34 -07:00
deps = [
":plasma_client",
":stats_lib",
2020-06-03 17:44:34 -07:00
],
)
FLATC_ARGS = [
"--gen-object-api",
"--gen-mutable",
"--scoped-enums",
]
flatbuffer_cc_library(
name = "plasma_fbs",
srcs = ["src/ray/object_manager/plasma/plasma.fbs"],
2020-06-03 17:44:34 -07:00
flatc_args = FLATC_ARGS,
out_prefix = "src/ray/object_manager/plasma/",
2020-06-03 17:44:34 -07:00
)
# === End of plasma definitions ===
cc_library(
name = "ray_mock",
hdrs = glob(
["src/mock/**/*.h"],
),
copts = COPTS,
strip_include_prefix = "src",
visibility = ["//visibility:public"],
)
[scheduling][5] Refactor resource syncer. (#23270) ## Why are these changes needed? This PR refactor the resource syncer to decouple it from GCS and raylet. GCS and raylet will use the same module to sync data. The integration will happen in the next PR. There are several new introduced components: * RaySyncer: the place where remote and local information sits. It's a coordinator layer. * NodeState: keeps track of the local status, similar to NodeSyncConnection. * NodeSyncConnection: keeps track of the sending and receiving information and make sure not sending the information the remote node knows. The core protocol is that each node will send {what it has} - {what the target has} to the target. For example, think about node A <-> B. A will send all A has exclude what B has to B. Whenever when there is new information (from NodeState or NodeSyncConnection), it'll be passed to RaySyncer broadcast message to broadcast. NodeSyncConnection is for the communication layer. It has two implementations Client and Server: * Server => Client: client will send a long-polling request and server will response every 100ms if there is data to be sent. * Client => Server: client will check every 100ms to see whether there is new data to be sent. If there is, just use RPC call to send the data. Here is one example: ```mermaid flowchart LR; A-->B; B-->C; B-->D; ``` It means A initialize the connection to B and B initialize the connections to C and D Now C generate a message M: 1. [C] RaySyncer check whether there is new message generated in C and get M 2. [C] RaySyncer will push M to NodeSyncConnection in local component (B) 3. [C] ServerSyncConnection will wait until B send a long polling and send the data to B 4. [B] B received the message from C and push it to local sync connection (C, A, D) 5. [B] ClientSyncConnection of C will not push it to its local queue since it's received by this channel. 6. [B] ClientSyncConnection of D will send this message to D 7. [B] ServerSyncConnection of A will be used to send this message to A (long-polling here) 8. [B] B will update NodeState (local component) with this message M 9. [D] D's pipelines is similar to 5) (with ServerSyncConnection) and 8) 10. [A] A's pipeline is similar to 5) and 8)
2022-03-29 23:52:39 -07:00
cc_grpc_library(
name = "ray_syncer_cc_grpc",
srcs = ["//src/ray/protobuf:ray_syncer_proto"],
grpc_only = True,
deps = ["//src/ray/protobuf:ray_syncer_cc_proto"],
)
cc_library(
name = "ray_common",
srcs = glob(
[
"src/ray/common/**/*.cc",
],
exclude = [
"src/ray/common/**/*_test.cc",
],
) + [
"src/ray/raylet/scheduling/cluster_resource_data.cc",
"src/ray/raylet/scheduling/fixed_point.cc",
"src/ray/raylet/scheduling/scheduling_ids.cc",
],
hdrs = glob(
[
"src/ray/common/*.h",
"src/ray/common/**/*.h",
],
) + [
"src/ray/raylet/scheduling/cluster_resource_data.h",
"src/ray/raylet/scheduling/fixed_point.h",
"src/ray/raylet/scheduling/scheduling_ids.h",
],
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
visibility = ["//visibility:public"],
deps = [
":node_manager_fbs",
[scheduling][5] Refactor resource syncer. (#23270) ## Why are these changes needed? This PR refactor the resource syncer to decouple it from GCS and raylet. GCS and raylet will use the same module to sync data. The integration will happen in the next PR. There are several new introduced components: * RaySyncer: the place where remote and local information sits. It's a coordinator layer. * NodeState: keeps track of the local status, similar to NodeSyncConnection. * NodeSyncConnection: keeps track of the sending and receiving information and make sure not sending the information the remote node knows. The core protocol is that each node will send {what it has} - {what the target has} to the target. For example, think about node A <-> B. A will send all A has exclude what B has to B. Whenever when there is new information (from NodeState or NodeSyncConnection), it'll be passed to RaySyncer broadcast message to broadcast. NodeSyncConnection is for the communication layer. It has two implementations Client and Server: * Server => Client: client will send a long-polling request and server will response every 100ms if there is data to be sent. * Client => Server: client will check every 100ms to see whether there is new data to be sent. If there is, just use RPC call to send the data. Here is one example: ```mermaid flowchart LR; A-->B; B-->C; B-->D; ``` It means A initialize the connection to B and B initialize the connections to C and D Now C generate a message M: 1. [C] RaySyncer check whether there is new message generated in C and get M 2. [C] RaySyncer will push M to NodeSyncConnection in local component (B) 3. [C] ServerSyncConnection will wait until B send a long polling and send the data to B 4. [B] B received the message from C and push it to local sync connection (C, A, D) 5. [B] ClientSyncConnection of C will not push it to its local queue since it's received by this channel. 6. [B] ClientSyncConnection of D will send this message to D 7. [B] ServerSyncConnection of A will be used to send this message to A (long-polling here) 8. [B] B will update NodeState (local component) with this message M 9. [D] D's pipelines is similar to 5) (with ServerSyncConnection) and 8) 10. [A] A's pipeline is similar to 5) and 8)
2022-03-29 23:52:39 -07:00
":ray_syncer_cc_grpc",
":ray_util",
":stats_metric",
"//src/ray/protobuf:common_cc_proto",
"//src/ray/protobuf:gcs_cc_proto",
"@boost//:asio",
"@com_github_grpc_grpc//:grpc++",
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/container:flat_hash_set",
[core] Reconstruction for lost plasma objects (#7733) * Add a lineage_ref_count to References * Refactor TaskManager to store TaskEntry as a struct * Refactor to fix deadlock between TaskManager and ReferenceCounter Add references to task specs * Pin TaskEntries and References in the lineage of any ObjectIDs in scope * Fix deadlock, convert num_plasma_returns to a set of object IDs * fix unit tests * Feature flag * Do not release lineage for objects that were promoted to plasma * fix build * fix build * Remove num executions * Remove num executions * Add pinned locations to ReferenceCounter, empty handler for node death * Fix num returns for actor tasks, fix Put return value * Add regression test * Clear pinned locations and callbacks on node removal * Clear pinned locations and callbacks on node removal * Simplify num return values * Remove unused * doc * tmp * Set num returns * Move lineage pinning flag to ReferenceCounter * comments * Recover from plasma failures by pinning a new copy * Basic object reconstruction, no concurrent reqs yet * reconstruction test suite and a few fixes: - fix for disabling lineage - fix for updating submitted task refs * Handle concurrent attempts to recover the same object * Fix deadlock in DrainAndShutdown * Revert "[core] Revert lineage pinning (#7499) (#7692)" This reverts commit ba86a02b375d1340ec77cf14a1bd099f58e8b0f4. * debug rllib * debug rllib * turn on all rllib tests again * debug rllib * Fix drain bug, check number of pending tasks * revert rllib debug * remove todo * Trigger rllib tests * revert rllib debug commit * Split out logic into ObjectRecoveryManager * Fix python tests * Refactor to remove dependency on gcs client * Unit tests * Move pinned at node ID to direct memory store * Unit test fixes and lint * simplify and more tests * Add ResubmitTask test for TaskManager * Doc * fix build * comments * Fix * debug * Update * fix * Fix * Fix bad status handling, unit test * Fix build
2020-04-11 16:52:57 -07:00
"@com_google_absl//absl/memory",
"@com_google_absl//absl/random",
"@com_google_absl//absl/synchronization",
"@com_google_absl//absl/time",
2019-11-15 16:04:16 -08:00
"@com_google_googletest//:gtest",
"@msgpack",
"@nlohmann_json",
],
)
# This header is used to warp some internal code so we can reduce suspicious
2020-09-09 16:53:43 +08:00
# symbols export.
cc_library(
name = "exported_internal",
2020-09-09 16:53:43 +08:00
srcs = glob(
[
"src/ray/internal/internal.cc",
2020-09-09 16:53:43 +08:00
],
),
hdrs = glob(
[
"src/ray/internal/internal.h",
2020-09-09 16:53:43 +08:00
],
),
copts = COPTS,
strip_include_prefix = "src",
visibility = ["//visibility:public"],
deps = [
":core_worker_lib",
],
alwayslink = 1,
)
2019-01-20 12:16:47 -08:00
cc_binary(
name = "raylet",
srcs = ["src/ray/raylet/main.cc"],
copts = COPTS,
visibility = ["//java:__subpackages__"],
2019-01-20 12:16:47 -08:00
deps = [
":ray_util",
":raylet_lib",
"@com_github_gflags_gflags//:gflags",
2019-01-20 12:16:47 -08:00
],
)
cc_library(
name = "gcs_pub_sub_lib",
srcs = [
"src/ray/gcs/pubsub/gcs_pub_sub.cc",
],
hdrs = [
"src/ray/gcs/pubsub/gcs_pub_sub.h",
],
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
deps = [
":gcs",
":gcs_service_rpc",
":pubsub_lib",
":ray_common",
":redis_client",
],
)
cc_library(
name = "gcs_server_lib",
srcs = glob(
[
"src/ray/gcs/gcs_server/*.cc",
],
exclude = [
"src/ray/gcs/gcs_server/gcs_server_main.cc",
"src/ray/gcs/gcs_server/test/*.cc",
],
),
hdrs = glob(
[
"src/ray/gcs/gcs_server/*.h",
],
),
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
deps = [
":agent_manager_rpc",
":gcs",
":gcs_pub_sub_lib",
":gcs_service_cc_grpc",
":gcs_service_rpc",
":gcs_table_storage_lib",
":node_manager_rpc",
":pubsub_lib",
":raylet_client_lib",
":scheduler",
":worker_rpc",
"@com_google_absl//absl/container:btree",
],
)
cc_binary(
name = "gcs_server",
srcs = [
"src/ray/gcs/gcs_server/gcs_server_main.cc",
],
copts = COPTS,
2020-02-05 12:06:25 +08:00
visibility = ["//java:__subpackages__"],
deps = [
":gcs_server_lib",
":stats_lib",
"@com_github_gflags_gflags//:gflags",
],
)
# Ray native pubsub module.
cc_library(
name = "pubsub_lib",
srcs = glob([
"src/ray/pubsub/*.cc",
]),
hdrs = glob([
"src/ray/pubsub/*.h",
]),
copts = COPTS,
strip_include_prefix = "src",
visibility = ["//visibility:public"],
deps = [
":pubsub_rpc",
"@boost//:any",
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/container:flat_hash_set",
"@com_google_absl//absl/synchronization",
],
)
cc_library(
name = "stats_metric",
srcs = [
"src/ray/stats/metric.cc",
"src/ray/stats/metric_defs.cc",
"src/ray/stats/tag_defs.cc",
],
hdrs = [
"src/ray/stats/metric.h",
"src/ray/stats/metric_defs.h",
"src/ray/stats/tag_defs.h",
],
strip_include_prefix = "src",
visibility = ["//visibility:public"],
deps = [
":ray_util",
"@com_github_jupp0r_prometheus_cpp//pull",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
"@com_google_googletest//:gtest",
"@io_opencensus_cpp//opencensus/exporters/stats/prometheus:prometheus_exporter",
"@io_opencensus_cpp//opencensus/exporters/stats/stdout:stdout_exporter",
"@io_opencensus_cpp//opencensus/stats",
"@io_opencensus_cpp//opencensus/tags",
],
)
cc_library(
name = "stats_lib",
srcs = [
"src/ray/stats/metric_exporter.cc",
"src/ray/stats/metric_exporter_client.cc",
],
hdrs = [
"src/ray/stats/metric.h",
"src/ray/stats/metric_exporter.h",
"src/ray/stats/metric_exporter_client.h",
"src/ray/stats/stats.h",
"src/ray/stats/tag_defs.h",
],
copts = COPTS,
linkopts = select({
"@bazel_tools//src/conditions:windows": [
],
"//conditions:default": [
"-lpthread",
],
}),
strip_include_prefix = "src",
visibility = ["//visibility:public"],
deps = [
":reporter_rpc",
":stats_metric",
],
)
cc_library(
name = "scheduler",
srcs = glob(
[
"src/ray/raylet/scheduling/**/*.cc",
],
exclude = [
"src/ray/raylet/scheduling/**/*_test.cc",
],
),
hdrs = glob(
[
"src/ray/raylet/scheduling/**/*.h",
"src/ray/core_worker/common.h",
],
),
copts = COPTS,
linkopts = select({
"@bazel_tools//src/conditions:windows": [
],
"//conditions:default": [
"-lpthread",
],
}),
strip_include_prefix = "src",
visibility = ["//visibility:public"],
deps = [
":gcs_client_lib",
":ray_common",
":ray_util",
":stats_lib",
"//src/ray/protobuf:common_cc_proto",
"@boost//:asio",
"@boost//:filesystem",
"@boost//:system",
"@com_github_jupp0r_prometheus_cpp//pull",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/container:flat_hash_set",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
"@com_google_googletest//:gtest",
"@io_opencensus_cpp//opencensus/exporters/stats/prometheus:prometheus_exporter",
"@io_opencensus_cpp//opencensus/stats",
"@io_opencensus_cpp//opencensus/tags",
],
)
2019-01-20 12:16:47 -08:00
cc_library(
name = "raylet_lib",
srcs = glob(
[
"src/ray/raylet/**/*.cc",
],
exclude = [
"src/ray/raylet/**/*_test.cc",
"src/ray/raylet/scheduling/**/*.cc",
"src/ray/raylet/main.cc",
],
),
hdrs = glob(
[
"src/ray/raylet/**/*.h",
],
exclude = [
"src/ray/raylet/scheduling/**/*.h",
"src/ray/raylet/main.cc",
],
),
copts = COPTS,
linkopts = select({
"@bazel_tools//src/conditions:windows": [
],
"//conditions:default": [
"-lpthread",
],
}),
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
visibility = ["//visibility:public"],
2019-01-20 12:16:47 -08:00
deps = [
":agent_manager_rpc",
2019-01-20 12:16:47 -08:00
":gcs",
":gcs_client_lib",
":node_manager_fbs",
":node_manager_rpc",
2019-01-20 12:16:47 -08:00
":object_manager",
2020-06-03 17:44:34 -07:00
":plasma_client",
":pubsub_lib",
2019-01-20 12:16:47 -08:00
":ray_common",
":ray_util",
":runtime_env_rpc",
":scheduler",
2019-04-03 12:01:02 +08:00
":stats_lib",
":worker_rpc",
"//src/ray/protobuf:common_cc_proto",
2019-01-20 12:16:47 -08:00
"@boost//:asio",
"@boost//:filesystem",
"@boost//:system",
2019-04-03 12:01:02 +08:00
"@com_github_jupp0r_prometheus_cpp//pull",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/container:flat_hash_set",
2019-04-03 12:01:02 +08:00
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
2019-01-20 12:16:47 -08:00
"@com_google_googletest//:gtest",
2019-04-03 12:01:02 +08:00
"@io_opencensus_cpp//opencensus/exporters/stats/prometheus:prometheus_exporter",
"@io_opencensus_cpp//opencensus/stats",
"@io_opencensus_cpp//opencensus/tags",
2019-01-20 12:16:47 -08:00
],
)
cc_library(
name = "raylet_client_lib",
srcs = glob([
"src/ray/raylet_client/*.cc",
]),
hdrs = glob([
"src/ray/raylet_client/*.h",
]),
copts = COPTS,
linkopts = select({
"@bazel_tools//src/conditions:windows": [
],
"//conditions:default": [
"-lpthread",
],
}),
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
visibility = ["//visibility:public"],
deps = [
":agent_manager_rpc",
":node_manager_fbs",
":node_manager_rpc",
":ray_common",
":ray_util",
"//src/ray/protobuf:gcs_cc_proto",
"@boost//:asio",
],
)
cc_library(
name = "core_worker_lib",
srcs = glob(
[
"src/ray/core_worker/*.cc",
"src/ray/core_worker/store_provider/*.cc",
2019-08-06 04:14:45 +08:00
"src/ray/core_worker/store_provider/memory_store/*.cc",
"src/ray/core_worker/transport/*.cc",
],
exclude = [
"src/ray/core_worker/**/*_test.cc",
"src/ray/core_worker/mock_worker.cc",
],
),
hdrs = glob([
"src/ray/core_worker/*.h",
"src/ray/core_worker/store_provider/*.h",
2019-08-06 04:14:45 +08:00
"src/ray/core_worker/store_provider/memory_store/*.h",
"src/ray/core_worker/transport/*.h",
]),
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
visibility = ["//visibility:public"],
deps = [
":gcs",
":gcs_client_lib",
":plasma_client",
":ray_common",
":ray_util",
":raylet_client_lib",
":stats_lib",
":worker_rpc",
"//src/ray/protobuf:worker_cc_proto",
"@boost//:fiber",
"@com_google_absl//absl/container:btree",
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/container:flat_hash_set",
],
)
2019-07-30 17:47:17 +08:00
cc_library(
name = "mock_worker_lib",
srcs = ["src/ray/core_worker/test/mock_worker.cc"],
hdrs = glob([
"src/ray/core_worker/test/*.h",
]),
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
deps = [
":core_worker_lib",
],
)
cc_binary(
2019-07-30 17:47:17 +08:00
name = "mock_worker",
copts = COPTS,
deps = [
":mock_worker_lib",
],
)
cc_test(
name = "core_worker_test",
size = "small",
2019-07-30 17:47:17 +08:00
srcs = ["src/ray/core_worker/test/core_worker_test.cc"],
args = [
"$(location raylet)",
"$(location mock_worker)",
"$(location gcs_server)",
"$(location redis-cli)",
"$(location redis-server)",
],
copts = COPTS,
data = [
"//:gcs_server",
"//:mock_worker",
"//:raylet",
"//:redis-cli",
"//:redis-server",
],
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":core_worker_lib",
":gcs",
"//src/ray/protobuf:runtime_env_common_cc_proto",
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/container:flat_hash_set",
"@com_google_googletest//:gtest",
],
)
cc_test(
name = "memory_store_test",
size = "small",
srcs = ["src/ray/core_worker/test/memory_store_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":core_worker_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "direct_actor_transport_test",
srcs = ["src/ray/core_worker/test/direct_actor_transport_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":core_worker_lib",
":ray_mock",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "direct_actor_transport_mock_test",
srcs = ["src/ray/core_worker/test/direct_actor_transport_mock_test.cc"],
copts = COPTS,
tags = ["team:core"],
deps = [
":core_worker_lib",
":ray_mock",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "gcs_function_manager_test",
srcs = ["src/ray/gcs/gcs_server/test/gcs_function_manager_test.cc"],
copts = COPTS,
tags = ["team:core"],
deps = [
":gcs_server_lib",
":ray_mock",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "direct_task_transport_test",
size = "small",
srcs = ["src/ray/core_worker/test/direct_task_transport_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":core_worker_lib",
":ray_mock",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "direct_task_transport_mock_test",
size = "small",
srcs = ["src/ray/core_worker/test/direct_task_transport_mock_test.cc"],
copts = COPTS,
tags = ["team:core"],
deps = [
":core_worker_lib",
":ray_mock",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "reference_count_test",
size = "small",
srcs = ["src/ray/core_worker/test/reference_count_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":core_worker_lib",
"@com_google_googletest//:gtest_main",
],
[core] Reconstruction for lost plasma objects (#7733) * Add a lineage_ref_count to References * Refactor TaskManager to store TaskEntry as a struct * Refactor to fix deadlock between TaskManager and ReferenceCounter Add references to task specs * Pin TaskEntries and References in the lineage of any ObjectIDs in scope * Fix deadlock, convert num_plasma_returns to a set of object IDs * fix unit tests * Feature flag * Do not release lineage for objects that were promoted to plasma * fix build * fix build * Remove num executions * Remove num executions * Add pinned locations to ReferenceCounter, empty handler for node death * Fix num returns for actor tasks, fix Put return value * Add regression test * Clear pinned locations and callbacks on node removal * Clear pinned locations and callbacks on node removal * Simplify num return values * Remove unused * doc * tmp * Set num returns * Move lineage pinning flag to ReferenceCounter * comments * Recover from plasma failures by pinning a new copy * Basic object reconstruction, no concurrent reqs yet * reconstruction test suite and a few fixes: - fix for disabling lineage - fix for updating submitted task refs * Handle concurrent attempts to recover the same object * Fix deadlock in DrainAndShutdown * Revert "[core] Revert lineage pinning (#7499) (#7692)" This reverts commit ba86a02b375d1340ec77cf14a1bd099f58e8b0f4. * debug rllib * debug rllib * turn on all rllib tests again * debug rllib * Fix drain bug, check number of pending tasks * revert rllib debug * remove todo * Trigger rllib tests * revert rllib debug commit * Split out logic into ObjectRecoveryManager * Fix python tests * Refactor to remove dependency on gcs client * Unit tests * Move pinned at node ID to direct memory store * Unit test fixes and lint * simplify and more tests * Add ResubmitTask test for TaskManager * Doc * fix build * comments * Fix * debug * Update * fix * Fix * Fix bad status handling, unit test * Fix build
2020-04-11 16:52:57 -07:00
)
cc_test(
name = "object_recovery_manager_test",
size = "small",
[core] Reconstruction for lost plasma objects (#7733) * Add a lineage_ref_count to References * Refactor TaskManager to store TaskEntry as a struct * Refactor to fix deadlock between TaskManager and ReferenceCounter Add references to task specs * Pin TaskEntries and References in the lineage of any ObjectIDs in scope * Fix deadlock, convert num_plasma_returns to a set of object IDs * fix unit tests * Feature flag * Do not release lineage for objects that were promoted to plasma * fix build * fix build * Remove num executions * Remove num executions * Add pinned locations to ReferenceCounter, empty handler for node death * Fix num returns for actor tasks, fix Put return value * Add regression test * Clear pinned locations and callbacks on node removal * Clear pinned locations and callbacks on node removal * Simplify num return values * Remove unused * doc * tmp * Set num returns * Move lineage pinning flag to ReferenceCounter * comments * Recover from plasma failures by pinning a new copy * Basic object reconstruction, no concurrent reqs yet * reconstruction test suite and a few fixes: - fix for disabling lineage - fix for updating submitted task refs * Handle concurrent attempts to recover the same object * Fix deadlock in DrainAndShutdown * Revert "[core] Revert lineage pinning (#7499) (#7692)" This reverts commit ba86a02b375d1340ec77cf14a1bd099f58e8b0f4. * debug rllib * debug rllib * turn on all rllib tests again * debug rllib * Fix drain bug, check number of pending tasks * revert rllib debug * remove todo * Trigger rllib tests * revert rllib debug commit * Split out logic into ObjectRecoveryManager * Fix python tests * Refactor to remove dependency on gcs client * Unit tests * Move pinned at node ID to direct memory store * Unit test fixes and lint * simplify and more tests * Add ResubmitTask test for TaskManager * Doc * fix build * comments * Fix * debug * Update * fix * Fix * Fix bad status handling, unit test * Fix build
2020-04-11 16:52:57 -07:00
srcs = ["src/ray/core_worker/test/object_recovery_manager_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
[core] Reconstruction for lost plasma objects (#7733) * Add a lineage_ref_count to References * Refactor TaskManager to store TaskEntry as a struct * Refactor to fix deadlock between TaskManager and ReferenceCounter Add references to task specs * Pin TaskEntries and References in the lineage of any ObjectIDs in scope * Fix deadlock, convert num_plasma_returns to a set of object IDs * fix unit tests * Feature flag * Do not release lineage for objects that were promoted to plasma * fix build * fix build * Remove num executions * Remove num executions * Add pinned locations to ReferenceCounter, empty handler for node death * Fix num returns for actor tasks, fix Put return value * Add regression test * Clear pinned locations and callbacks on node removal * Clear pinned locations and callbacks on node removal * Simplify num return values * Remove unused * doc * tmp * Set num returns * Move lineage pinning flag to ReferenceCounter * comments * Recover from plasma failures by pinning a new copy * Basic object reconstruction, no concurrent reqs yet * reconstruction test suite and a few fixes: - fix for disabling lineage - fix for updating submitted task refs * Handle concurrent attempts to recover the same object * Fix deadlock in DrainAndShutdown * Revert "[core] Revert lineage pinning (#7499) (#7692)" This reverts commit ba86a02b375d1340ec77cf14a1bd099f58e8b0f4. * debug rllib * debug rllib * turn on all rllib tests again * debug rllib * Fix drain bug, check number of pending tasks * revert rllib debug * remove todo * Trigger rllib tests * revert rllib debug commit * Split out logic into ObjectRecoveryManager * Fix python tests * Refactor to remove dependency on gcs client * Unit tests * Move pinned at node ID to direct memory store * Unit test fixes and lint * simplify and more tests * Add ResubmitTask test for TaskManager * Doc * fix build * comments * Fix * debug * Update * fix * Fix * Fix bad status handling, unit test * Fix build
2020-04-11 16:52:57 -07:00
deps = [
":core_worker_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "scheduling_queue_test",
srcs = ["src/ray/core_worker/test/scheduling_queue_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":core_worker_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "concurrency_group_manager_test",
srcs = ["src/ray/core_worker/test/concurrency_group_manager_test.cc"],
copts = COPTS,
tags = ["team:core"],
deps = [
":core_worker_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "actor_submit_queue_test",
size = "small",
srcs = ["src/ray/core_worker/test/actor_submit_queue_test.cc"],
copts = COPTS,
tags = ["team:core"],
deps = [
":core_worker_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "task_manager_test",
size = "small",
srcs = ["src/ray/core_worker/test/task_manager_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":core_worker_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "actor_creator_test",
size = "small",
srcs = ["src/ray/core_worker/test/actor_creator_test.cc"],
copts = COPTS,
tags = ["team:core"],
deps = [
":core_worker_lib",
":gcs_client_lib",
":ray_mock",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "actor_manager_test",
size = "small",
srcs = ["src/ray/core_worker/test/actor_manager_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":core_worker_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "lease_policy_test",
size = "small",
srcs = ["src/ray/core_worker/test/lease_policy_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":core_worker_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "resource_request_test",
size = "small",
srcs = [
"src/ray/raylet/scheduling/resource_request_test.cc",
],
copts = COPTS,
tags = ["team:core"],
deps = [
":scheduler",
"@com_google_googletest//:gtest_main",
],
)
2019-11-15 16:04:16 -08:00
cc_test(
name = "cluster_resource_scheduler_test",
size = "small",
srcs = [
"src/ray/raylet/scheduling/cluster_resource_scheduler_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":ray_mock",
":raylet_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "cluster_resource_scheduler_2_test",
size = "small",
srcs = [
"src/ray/raylet/scheduling/cluster_resource_scheduler_2_test.cc",
],
copts = COPTS,
tags = ["team:core"],
deps = [
":raylet_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "scheduling_policy_test",
size = "small",
srcs = [
"src/ray/raylet/scheduling/policy/scheduling_policy_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":raylet_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "cluster_task_manager_test",
size = "small",
srcs = [
"src/ray/raylet/scheduling/cluster_task_manager_test.cc",
],
2019-11-15 16:04:16 -08:00
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
2019-11-15 16:04:16 -08:00
deps = [
":ray_mock",
":raylet_lib",
2019-11-15 16:04:16 -08:00
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "cluster_resource_manager_test",
size = "small",
srcs = [
"src/ray/raylet/scheduling/cluster_resource_manager_test.cc",
],
copts = COPTS,
tags = ["team:core"],
deps = [
":raylet_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "scheduling_ids_test",
size = "small",
srcs = [
"src/ray/raylet/scheduling/scheduling_ids_test.cc",
],
copts = COPTS,
tags = ["team:core"],
deps = [
":raylet_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "local_object_manager_test",
size = "small",
srcs = [
"src/ray/raylet/test/local_object_manager_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":raylet_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "pull_manager_test",
size = "small",
srcs = [
"src/ray/object_manager/test/pull_manager_test.cc",
],
copts = COPTS,
tags = ["team:core"],
deps = [
":object_manager",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "object_buffer_pool_test",
size = "small",
srcs = [
"src/ray/object_manager/test/object_buffer_pool_test.cc",
],
copts = COPTS,
tags = ["team:core"],
deps = [
":object_manager",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "ownership_based_object_directory_test",
size = "small",
srcs = [
"src/ray/object_manager/test/ownership_based_object_directory_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":object_manager",
":ray_mock",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "push_manager_test",
size = "small",
srcs = [
"src/ray/object_manager/test/push_manager_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":raylet_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "spilled_object_test",
size = "small",
srcs = [
"src/ray/object_manager/test/spilled_object_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":raylet_lib",
"@boost//:endian",
"@com_google_absl//absl/strings:str_format",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "fallback_allocator_test",
srcs = [
"src/ray/object_manager/plasma/test/fallback_allocator_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":plasma_store_server_lib",
"@boost//:filesystem",
"@com_google_absl//absl/strings:str_format",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "object_store_test",
srcs = [
"src/ray/object_manager/plasma/test/object_store_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":plasma_store_server_lib",
"@com_google_absl//absl/random",
"@com_google_absl//absl/strings:str_format",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "object_lifecycle_manager_test",
srcs = [
"src/ray/object_manager/plasma/test/object_lifecycle_manager_test.cc",
"src/ray/object_manager/plasma/test/stats_collector_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":plasma_store_server_lib",
"@com_google_absl//absl/random",
"@com_google_absl//absl/strings:str_format",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "eviction_policy_test",
srcs = [
"src/ray/object_manager/plasma/test/eviction_policy_test.cc",
],
copts = COPTS,
tags = ["team:core"],
deps = [
":plasma_store_server_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "create_request_queue_test",
size = "small",
srcs = [
"src/ray/object_manager/test/create_request_queue_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":plasma_store_server_lib",
"@com_google_googletest//:gtest",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "get_request_queue_test",
size = "small",
srcs = [
"src/ray/object_manager/test/get_request_queue_test.cc",
],
copts = COPTS,
tags = ["team:core"],
deps = [
":plasma_store_server_lib",
"@com_google_googletest//:gtest",
"@com_google_googletest//:gtest_main",
],
)
2019-01-20 12:16:47 -08:00
cc_test(
name = "worker_pool_test",
size = "small",
2019-01-20 12:16:47 -08:00
srcs = ["src/ray/raylet/worker_pool_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
2019-01-20 12:16:47 -08:00
deps = [
":raylet_lib",
"@com_google_googletest//:gtest_main",
2019-01-20 12:16:47 -08:00
],
)
cc_test(
name = "gcs_placement_group_manager_mock_test",
size = "small",
srcs = [
"src/ray/gcs/gcs_server/test/gcs_placement_group_manager_mock_test.cc",
],
copts = COPTS,
tags = ["team:core"],
deps = [
":gcs_server_lib",
":gcs_test_util_lib",
":ray_mock",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "placement_group_resource_manager_test",
size = "small",
srcs = ["src/ray/raylet/placement_group_resource_manager_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
"gcs_test_util_lib",
"ray_common",
"raylet_lib",
":ray_mock",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "asio_defer_test",
size = "small",
srcs = ["src/ray/common/test/asio_defer_test.cc"],
copts = COPTS,
tags = ["team:core"],
deps = [
"ray_common",
"@com_google_googletest//:gtest_main",
],
)
[ID Refactor] Refactor ActorID, TaskID and ObjectID (#5286) * Refactor ActorID, TaskID on the Java side. Left a TODO comment WIP for ObjectID ADD test Fix Add java part Fix Java test Fix Refine test. Enable test in CI * Extra a helper function. * Resolve TODOs * Fix Python CI * Fix Java lint * Update .travis.yml Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Address some comments. Address some comments. Add id_specification.rst Reanme id_specification.rst to id_specification.md typo Address zhijun's comments. Fix test Address comments. Fix lint Address comments * Fix test * Address comments. * Fix build error * Update src/ray/design_docs/id_specification.md Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/design_docs/id_specification.md Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/design_docs/id_specification.md Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/design_docs/id_specification.md Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/design_docs/id_specification.md Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Address comments * Update src/ray/common/id.h Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/common/id.h Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/common/id.h Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/design_docs/id_specification.md Co-Authored-By: Hao Chen <chenh1024@gmail.com> * Update src/ray/design_docs/id_specification.md Co-Authored-By: Hao Chen <chenh1024@gmail.com> * Address comments. * Address comments. * Address comments. * Update C++ part to make sure task id is generated determantic * WIP * Fix core worker * Fix Java part * Fix comments. * Add Python side * Fix python * Address comments * Fix linting * Fix * Fix C++ linting * Add JobId() method to TaskID * Fix linting * Update src/ray/common/id.h Co-Authored-By: Hao Chen <chenh1024@gmail.com> * Update java/api/src/main/java/org/ray/api/id/TaskId.java Co-Authored-By: Hao Chen <chenh1024@gmail.com> * Update java/api/src/main/java/org/ray/api/id/TaskId.java Co-Authored-By: Hao Chen <chenh1024@gmail.com> * Update java/api/src/main/java/org/ray/api/id/ActorId.java Co-Authored-By: Hao Chen <chenh1024@gmail.com> * Address comments * Add DriverTaskId embeding job id * Fix tests * Add python dor_fake_driver_id * Address comments and fix linting * Fix CI
2019-08-07 11:04:51 +08:00
cc_test(
name = "id_test",
size = "small",
[ID Refactor] Refactor ActorID, TaskID and ObjectID (#5286) * Refactor ActorID, TaskID on the Java side. Left a TODO comment WIP for ObjectID ADD test Fix Add java part Fix Java test Fix Refine test. Enable test in CI * Extra a helper function. * Resolve TODOs * Fix Python CI * Fix Java lint * Update .travis.yml Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Address some comments. Address some comments. Add id_specification.rst Reanme id_specification.rst to id_specification.md typo Address zhijun's comments. Fix test Address comments. Fix lint Address comments * Fix test * Address comments. * Fix build error * Update src/ray/design_docs/id_specification.md Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/design_docs/id_specification.md Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/design_docs/id_specification.md Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/design_docs/id_specification.md Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/design_docs/id_specification.md Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Address comments * Update src/ray/common/id.h Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/common/id.h Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/common/id.h Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/design_docs/id_specification.md Co-Authored-By: Hao Chen <chenh1024@gmail.com> * Update src/ray/design_docs/id_specification.md Co-Authored-By: Hao Chen <chenh1024@gmail.com> * Address comments. * Address comments. * Address comments. * Update C++ part to make sure task id is generated determantic * WIP * Fix core worker * Fix Java part * Fix comments. * Add Python side * Fix python * Address comments * Fix linting * Fix * Fix C++ linting * Add JobId() method to TaskID * Fix linting * Update src/ray/common/id.h Co-Authored-By: Hao Chen <chenh1024@gmail.com> * Update java/api/src/main/java/org/ray/api/id/TaskId.java Co-Authored-By: Hao Chen <chenh1024@gmail.com> * Update java/api/src/main/java/org/ray/api/id/TaskId.java Co-Authored-By: Hao Chen <chenh1024@gmail.com> * Update java/api/src/main/java/org/ray/api/id/ActorId.java Co-Authored-By: Hao Chen <chenh1024@gmail.com> * Address comments * Add DriverTaskId embeding job id * Fix tests * Add python dor_fake_driver_id * Address comments and fix linting * Fix CI
2019-08-07 11:04:51 +08:00
srcs = ["src/ray/common/id_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
[ID Refactor] Refactor ActorID, TaskID and ObjectID (#5286) * Refactor ActorID, TaskID on the Java side. Left a TODO comment WIP for ObjectID ADD test Fix Add java part Fix Java test Fix Refine test. Enable test in CI * Extra a helper function. * Resolve TODOs * Fix Python CI * Fix Java lint * Update .travis.yml Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Address some comments. Address some comments. Add id_specification.rst Reanme id_specification.rst to id_specification.md typo Address zhijun's comments. Fix test Address comments. Fix lint Address comments * Fix test * Address comments. * Fix build error * Update src/ray/design_docs/id_specification.md Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/design_docs/id_specification.md Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/design_docs/id_specification.md Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/design_docs/id_specification.md Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/design_docs/id_specification.md Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Address comments * Update src/ray/common/id.h Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/common/id.h Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/common/id.h Co-Authored-By: Stephanie Wang <swang@cs.berkeley.edu> * Update src/ray/design_docs/id_specification.md Co-Authored-By: Hao Chen <chenh1024@gmail.com> * Update src/ray/design_docs/id_specification.md Co-Authored-By: Hao Chen <chenh1024@gmail.com> * Address comments. * Address comments. * Address comments. * Update C++ part to make sure task id is generated determantic * WIP * Fix core worker * Fix Java part * Fix comments. * Add Python side * Fix python * Address comments * Fix linting * Fix * Fix C++ linting * Add JobId() method to TaskID * Fix linting * Update src/ray/common/id.h Co-Authored-By: Hao Chen <chenh1024@gmail.com> * Update java/api/src/main/java/org/ray/api/id/TaskId.java Co-Authored-By: Hao Chen <chenh1024@gmail.com> * Update java/api/src/main/java/org/ray/api/id/TaskId.java Co-Authored-By: Hao Chen <chenh1024@gmail.com> * Update java/api/src/main/java/org/ray/api/id/ActorId.java Co-Authored-By: Hao Chen <chenh1024@gmail.com> * Address comments * Add DriverTaskId embeding job id * Fix tests * Add python dor_fake_driver_id * Address comments and fix linting * Fix CI
2019-08-07 11:04:51 +08:00
deps = [
"ray_common",
"@com_google_googletest//:gtest_main",
],
)
2019-01-20 12:16:47 -08:00
cc_test(
name = "logging_test",
size = "small",
2019-01-20 12:16:47 -08:00
srcs = ["src/ray/util/logging_test.cc"],
args = ["--gtest_filter=PrintLogTest*"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
2019-01-20 12:16:47 -08:00
deps = [
":ray_util",
"@boost//:asio",
"@com_google_absl//absl/strings:str_format",
"@com_google_googletest//:gtest_main",
2019-01-20 12:16:47 -08:00
],
)
cc_test(
name = "event_test",
size = "small",
srcs = ["src/ray/util/event_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":ray_util",
2020-09-16 11:54:05 +08:00
"@boost//:filesystem",
"@boost//:range",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "filesystem_test",
size = "small",
srcs = ["src/ray/util/filesystem_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":ray_util",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "container_util_test",
size = "small",
srcs = ["src/ray/util/container_util_test.cc"],
copts = COPTS,
tags = ["team:core"],
deps = [
":ray_util",
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/container:flat_hash_set",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "util_test",
size = "small",
srcs = ["src/ray/util/util_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":ray_util",
"@boost//:asio",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "throttler_test",
size = "small",
srcs = ["src/ray/util/throttler_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":ray_util",
"@com_google_absl//absl/time",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "sample_test",
size = "small",
srcs = ["src/ray/util/sample_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":ray_common",
"@com_google_googletest//:gtest_main",
],
)
2019-01-20 12:16:47 -08:00
cc_test(
name = "dependency_manager_test",
size = "small",
srcs = ["src/ray/raylet/dependency_manager_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
2019-01-20 12:16:47 -08:00
deps = [
":raylet_lib",
2019-01-20 12:16:47 -08:00
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "wait_manager_test",
size = "small",
srcs = ["src/ray/raylet/wait_manager_test.cc"],
copts = COPTS,
tags = ["team:core"],
deps = [
":raylet_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "client_connection_test",
size = "small",
srcs = ["src/ray/common/test/client_connection_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":ray_common",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "status_test",
size = "small",
srcs = [
"src/ray/common/test/status_test.cc",
],
copts = COPTS,
tags = ["team:core"],
deps = [
":ray_common",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "pubsub_integration_test",
timeout = "short",
srcs = ["src/ray/pubsub/test/integration_test.cc"],
copts = COPTS,
tags = ["team:core"],
deps = [
":pubsub_cc_grpc",
":pubsub_lib",
"//src/ray/protobuf:pubsub_cc_proto",
"@com_google_absl//absl/synchronization",
"@com_google_absl//absl/time",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "publisher_test",
size = "small",
srcs = ["src/ray/pubsub/test/publisher_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":pubsub_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "subscriber_test",
size = "small",
srcs = [
"src/ray/pubsub/test/subscriber_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":pubsub_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "signal_test",
size = "small",
srcs = ["src/ray/util/signal_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":raylet_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "sequencer_test",
size = "small",
srcs = ["src/ray/util/sequencer_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":ray_util",
"@com_google_googletest//:gtest_main",
],
)
2019-04-03 12:01:02 +08:00
cc_test(
name = "stats_test",
size = "small",
2019-04-03 12:01:02 +08:00
srcs = ["src/ray/stats/stats_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = [
"stats",
"team:core",
],
2019-04-03 12:01:02 +08:00
deps = [
":stats_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "metric_exporter_client_test",
size = "small",
srcs = ["src/ray/stats/metric_exporter_client_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = [
"stats",
"team:core",
],
deps = [
":stats_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "gcs_test_util_lib",
hdrs = [
"src/ray/gcs/test/gcs_test_util.h",
],
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
deps = [
":gcs",
":gcs_service_rpc",
],
)
cc_test(
name = "grpc_server_client_test",
size = "small",
srcs = [
"src/ray/rpc/test/grpc_server_client_test.cc",
],
copts = COPTS,
tags = ["team:core"],
deps = [
":grpc_common_lib",
":test_service_cc_grpc",
],
)
cc_test(
name = "gcs_server_rpc_test",
size = "small",
srcs = [
"src/ray/gcs/gcs_server/test/gcs_server_rpc_test.cc",
],
args = [
"$(location redis-server)",
"$(location redis-cli)",
],
copts = COPTS,
data = [
"//:redis-cli",
"//:redis-server",
],
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":gcs_server_lib",
":gcs_test_util_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "gcs_kv_manager_test",
size = "small",
srcs = [
"src/ray/gcs/gcs_server/test/gcs_kv_manager_test.cc",
],
args = [
"$(location redis-server)",
"$(location redis-cli)",
],
copts = COPTS,
data = [
"//:redis-cli",
"//:redis-server",
],
tags = ["team:core"],
deps = [
":gcs_server_lib",
":gcs_test_util_lib",
"@com_google_googletest//:gtest_main",
],
)
2020-07-19 12:53:08 -07:00
cc_library(
name = "gcs_server_test_util",
hdrs = [
"src/ray/gcs/gcs_server/test/gcs_server_test_util.h",
],
copts = COPTS,
strip_include_prefix = "src",
deps = [
":gcs_client_lib",
],
2020-07-19 12:53:08 -07:00
)
cc_test(
name = "gcs_node_manager_test",
size = "small",
srcs = [
"src/ray/gcs/gcs_server/test/gcs_node_manager_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":gcs_server_lib",
2020-07-19 12:53:08 -07:00
":gcs_server_test_util",
":gcs_test_util_lib",
":ray_mock",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "gcs_job_manager_test",
size = "small",
srcs = [
"src/ray/gcs/gcs_server/test/gcs_job_manager_test.cc",
],
copts = COPTS,
tags = ["team:core"],
deps = [
":gcs_server_lib",
":gcs_server_test_util",
":gcs_test_util_lib",
":ray_mock",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "gcs_placement_group_manager_test",
size = "small",
srcs = [
"src/ray/gcs/gcs_server/test/gcs_placement_group_manager_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":gcs_server_lib",
2020-07-19 12:53:08 -07:00
":gcs_server_test_util",
":gcs_test_util_lib",
":ray_mock",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "gcs_placement_group_scheduler_test",
size = "small",
srcs = [
"src/ray/gcs/gcs_server/test/gcs_placement_group_scheduler_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":gcs_server_lib",
2020-07-19 12:53:08 -07:00
":gcs_server_test_util",
":gcs_test_util_lib",
":ray_mock",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "raylet_based_actor_scheduler_test",
size = "small",
srcs = [
"src/ray/gcs/gcs_server/test/raylet_based_actor_scheduler_test.cc",
],
copts = COPTS,
tags = ["team:core"],
deps = [
":gcs_client_lib",
":gcs_server_lib",
":gcs_server_test_util",
":gcs_test_util_lib",
":ray_mock",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "gcs_actor_scheduler_mock_test",
size = "small",
srcs = [
"src/ray/gcs/gcs_server/test/gcs_actor_scheduler_mock_test.cc",
],
copts = COPTS,
tags = ["team:core"],
deps = [
":gcs_server_lib",
":ray_mock",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "gcs_based_actor_scheduler_test",
size = "small",
srcs = [
"src/ray/gcs/gcs_server/test/gcs_based_actor_scheduler_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":gcs_server_lib",
2020-07-19 12:53:08 -07:00
":gcs_server_test_util",
":gcs_test_util_lib",
":ray_mock",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "gcs_actor_manager_test",
size = "small",
srcs = [
"src/ray/gcs/gcs_server/test/gcs_actor_manager_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":gcs_server_lib",
2020-07-19 12:53:08 -07:00
":gcs_server_test_util",
":gcs_test_util_lib",
":ray_mock",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "gcs_table_storage_lib",
srcs = glob(
[
"src/ray/gcs/gcs_server/gcs_table_storage.cc",
],
),
hdrs = glob(
[
"src/ray/gcs/gcs_server/gcs_table_storage.h",
],
),
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
deps = [
":gcs",
":gcs_in_memory_store_client",
":pubsub_lib",
":ray_common",
":redis_store_client",
],
)
cc_library(
name = "gcs_table_storage_test_lib",
hdrs = [
"src/ray/gcs/gcs_server/test/gcs_table_storage_test_base.h",
],
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
deps = [
"redis_store_client",
],
)
cc_test(
name = "redis_gcs_table_storage_test",
size = "small",
srcs = [
"src/ray/gcs/gcs_server/test/redis_gcs_table_storage_test.cc",
],
args = [
"$(location redis-server)",
"$(location redis-cli)",
],
copts = COPTS,
data = [
"//:redis-cli",
"//:redis-server",
],
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":gcs_table_storage_lib",
":gcs_table_storage_test_lib",
":gcs_test_util_lib",
":store_client_test_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "in_memory_gcs_table_storage_test",
size = "small",
srcs = [
"src/ray/gcs/gcs_server/test/in_memory_gcs_table_storage_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":gcs_table_storage_lib",
":gcs_table_storage_test_lib",
":gcs_test_util_lib",
":store_client_test_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "gcs_resource_manager_test",
size = "small",
srcs = [
"src/ray/gcs/gcs_server/test/gcs_resource_manager_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":gcs_server_lib",
":gcs_test_util_lib",
"@com_google_googletest//:gtest_main",
],
)
2021-03-29 11:36:30 -07:00
cc_test(
name = "gcs_resource_report_poller_test",
size = "small",
2021-03-29 11:36:30 -07:00
srcs = [
"src/ray/gcs/gcs_server/test/gcs_resource_report_poller_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
2021-03-29 11:36:30 -07:00
deps = [
":gcs_server_lib",
":gcs_test_util_lib",
"@com_google_googletest//:gtest_main",
],
)
2021-05-05 11:20:08 -07:00
cc_test(
name = "grpc_based_resource_broadcaster_test",
size = "small",
2021-05-05 11:20:08 -07:00
srcs = [
"src/ray/gcs/gcs_server/test/grpc_based_resource_broadcaster_test.cc",
],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
2021-05-05 11:20:08 -07:00
deps = [
":gcs_server_lib",
":gcs_test_util_lib",
"@com_google_googletest//:gtest_main",
],
)
2020-02-05 12:06:25 +08:00
cc_library(
name = "gcs_client_lib",
srcs = [
"src/ray/gcs/gcs_client/accessor.cc",
"src/ray/gcs/gcs_client/gcs_client.cc",
],
hdrs = [
"src/ray/gcs/gcs_client/accessor.h",
"src/ray/gcs/gcs_client/gcs_client.h",
],
2020-02-05 12:06:25 +08:00
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
2020-02-05 12:06:25 +08:00
deps = [
":gcs",
":gcs_pub_sub_lib",
":gcs_service_rpc",
":pubsub_lib",
":redis_store_client",
2020-02-05 12:06:25 +08:00
],
)
cc_library(
name = "global_state_accessor_lib",
srcs = glob(
[
"src/ray/gcs/gcs_client/global_state_accessor.cc",
],
),
hdrs = glob(
[
"src/ray/gcs/gcs_client/global_state_accessor.h",
],
),
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
visibility = ["//visibility:public"],
deps = [
":gcs_client_lib",
],
)
cc_test(
name = "accessor_test",
size = "small",
srcs = [
"src/ray/gcs/gcs_client/test/accessor_test.cc",
],
copts = COPTS,
tags = ["team:core"],
deps = [
":gcs_client_lib",
":gcs_test_util_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "global_state_accessor_test",
size = "small",
srcs = [
"src/ray/gcs/gcs_client/test/global_state_accessor_test.cc",
],
args = [
"$(location redis-server)",
"$(location redis-cli)",
],
copts = COPTS,
data = [
"//:redis-cli",
"//:redis-server",
],
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":gcs_client_lib",
":gcs_server_lib",
":gcs_test_util_lib",
":global_state_accessor_lib",
"@com_google_googletest//:gtest_main",
],
)
2020-02-05 12:06:25 +08:00
cc_test(
name = "gcs_client_test",
srcs = [
"src/ray/gcs/gcs_client/test/gcs_client_test.cc",
],
args = [
"$(location redis-server)",
"$(location redis-cli)",
],
2020-02-05 12:06:25 +08:00
copts = COPTS,
data = [
"//:redis-cli",
"//:redis-server",
],
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
2020-02-05 12:06:25 +08:00
deps = [
":gcs_client_lib",
":gcs_server_lib",
":gcs_test_util_lib",
2020-02-05 12:06:25 +08:00
"@com_google_googletest//:gtest_main",
],
)
2019-01-20 12:16:47 -08:00
cc_library(
name = "object_manager",
srcs = glob([
"src/ray/object_manager/*.cc",
"src/ray/object_manager/notification/*.cc",
2019-01-20 12:16:47 -08:00
]),
hdrs = glob([
"src/ray/object_manager/*.h",
"src/ray/object_manager/notification/*.h",
2019-01-20 12:16:47 -08:00
]),
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
2019-01-20 12:16:47 -08:00
deps = [
":core_worker_lib",
2019-01-20 12:16:47 -08:00
":gcs",
":object_manager_rpc",
":plasma_store_server_lib",
2019-01-20 12:16:47 -08:00
":ray_common",
":ray_util",
"@boost//:asio",
],
)
cc_library(
name = "platform_shims",
srcs = [] + select({
"@bazel_tools//src/conditions:windows": glob([
"src/shims/windows/**/*.c",
"src/shims/windows/**/*.cc",
"src/shims/windows/**/*.h",
]),
"//conditions:default": [],
}),
hdrs = [] + select({
"@bazel_tools//src/conditions:windows": glob([
"src/shims/windows/**/*.h",
]),
"//conditions:default": [],
}),
copts = COPTS,
strip_include_prefix = select({
"@bazel_tools//src/conditions:windows": "src/shims/windows",
"//conditions:default": "",
}),
visibility = ["//visibility:public"],
)
2019-01-20 12:16:47 -08:00
cc_library(
name = "ray_util",
srcs = glob(
[
"src/ray/util/*.cc",
],
exclude = [
"src/ray/util/*_test.cc",
],
),
2019-01-20 12:16:47 -08:00
hdrs = glob([
"src/ray/util/*.h",
]),
copts = COPTS,
linkopts = select({
"@bazel_tools//src/conditions:windows": [
],
"//conditions:default": [
"-lpthread",
],
}),
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
visibility = ["//visibility:public"],
2019-01-20 12:16:47 -08:00
deps = [
":aligned_alloc",
":sha256",
"//src/ray/protobuf:event_cc_proto",
"@boost//:asio",
"@boost//:filesystem",
2020-09-16 11:54:05 +08:00
"@com_github_spdlog//:spdlog",
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/debugging:failure_signal_handler",
"@com_google_absl//absl/debugging:stacktrace",
"@com_google_absl//absl/debugging:symbolize",
"@com_google_absl//absl/random",
"@com_google_absl//absl/synchronization",
"@com_google_absl//absl/time",
"@com_google_googletest//:gtest_main",
"@nlohmann_json",
2019-01-27 18:32:04 -08:00
],
2019-01-20 12:16:47 -08:00
)
filegroup(
name = "extra_actions_base_proto",
srcs = [
"thirdparty/protobuf/extra_actions_base.proto",
],
)
proto_library(
name = "extra_actions_base_proto_lib",
srcs = ["thirdparty/protobuf/extra_actions_base.proto"],
)
cc_proto_library(
name = "extra_actions_cc_proto_lib",
deps = [":extra_actions_base_proto_lib"],
)
filegroup(
name = "iwyu_sh",
srcs = ["ci/run/iwyu.sh"],
)
action_listener(
name = "iwyu_cpp",
extra_actions = [":iwyu_action"],
mnemonics = ["CppCompile"],
)
extra_action(
name = "iwyu_action",
cmd = "$(location :iwyu_sh) postbuild $(location @com_google_protobuf//:protoc) $(location :extra_actions_base_proto) --extra_action_file=$(EXTRA_ACTION_FILE) > $(output $(ACTION_ID).iwyu.txt)",
out_templates = [
"$(ACTION_ID).iwyu.txt",
],
tools = [
":extra_actions_base_proto",
":iwyu_sh",
"@com_google_protobuf//:protoc",
],
)
2019-01-20 12:16:47 -08:00
cc_library(
name = "sha256",
srcs = [
"src/ray/thirdparty/sha256.c",
],
hdrs = [
"src/ray/thirdparty/sha256.h",
],
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
)
cc_library(
name = "aligned_alloc",
srcs = [
"src/ray/thirdparty/aligned_alloc.c",
],
hdrs = [
"src/ray/thirdparty/aligned_alloc.h",
],
copts = COPTS,
strip_include_prefix = "src",
2019-01-20 12:16:47 -08:00
)
alias(
name = "hiredis",
actual = "@com_github_redis_hiredis//:hiredis",
)
cc_library(
name = "redis_client",
srcs = [
"src/ray/gcs/asio.cc",
"src/ray/gcs/redis_async_context.cc",
"src/ray/gcs/redis_client.cc",
"src/ray/gcs/redis_context.cc",
],
hdrs = [
"src/ray/gcs/asio.h",
"src/ray/gcs/redis_async_context.h",
"src/ray/gcs/redis_client.h",
"src/ray/gcs/redis_context.h",
],
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
deps = [
":hiredis",
":ray_common",
":ray_util",
":stats_lib",
"@boost//:asio",
],
)
cc_library(
name = "redis_store_client",
srcs = [
"src/ray/gcs/store_client/redis_store_client.cc",
],
hdrs = [
"src/ray/gcs/callback.h",
"src/ray/gcs/store_client/redis_store_client.h",
"src/ray/gcs/store_client/store_client.h",
],
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
deps = [
"redis_client",
],
)
cc_library(
name = "gcs_in_memory_store_client",
srcs = [
"src/ray/gcs/store_client/in_memory_store_client.cc",
],
hdrs = [
"src/ray/gcs/callback.h",
"src/ray/gcs/store_client/in_memory_store_client.h",
"src/ray/gcs/store_client/store_client.h",
],
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
deps = [
":ray_common",
":ray_util",
],
)
cc_library(
name = "store_client_test_lib",
hdrs = [
"src/ray/gcs/store_client/test/store_client_test_base.h",
],
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
deps = [
"redis_store_client",
],
)
cc_test(
name = "redis_store_client_test",
size = "small",
srcs = ["src/ray/gcs/store_client/test/redis_store_client_test.cc"],
args = [
"$(location redis-server)",
"$(location redis-cli)",
],
copts = COPTS,
data = [
"//:redis-cli",
"//:redis-server",
],
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":redis_store_client",
":store_client_test_lib",
"@com_google_googletest//:gtest_main",
],
)
cc_test(
name = "in_memory_store_client_test",
size = "small",
srcs = ["src/ray/gcs/store_client/test/in_memory_store_client_test.cc"],
copts = COPTS,
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
deps = [
":gcs_in_memory_store_client",
":store_client_test_lib",
"@com_google_googletest//:gtest_main",
],
)
2019-01-20 12:16:47 -08:00
cc_library(
name = "gcs",
srcs = glob(
[
"src/ray/gcs/*.cc",
],
exclude = [
"src/ray/gcs/*_test.cc",
],
),
2019-01-20 12:16:47 -08:00
hdrs = glob([
"src/ray/gcs/*.h",
]),
copts = COPTS,
2020-07-19 12:53:08 -07:00
strip_include_prefix = "src",
2019-01-20 12:16:47 -08:00
deps = [
":agent_manager_rpc",
":hiredis",
":node_manager_fbs",
":node_manager_rpc",
2019-01-20 12:16:47 -08:00
":ray_common",
":ray_util",
2019-04-03 12:01:02 +08:00
":stats_lib",
"//src/ray/protobuf:gcs_cc_proto",
"//src/ray/protobuf:gcs_service_cc_proto",
2019-01-20 12:16:47 -08:00
"@boost//:asio",
],
)
2021-09-24 20:11:43 -07:00
[scheduling][5] Refactor resource syncer. (#23270) ## Why are these changes needed? This PR refactor the resource syncer to decouple it from GCS and raylet. GCS and raylet will use the same module to sync data. The integration will happen in the next PR. There are several new introduced components: * RaySyncer: the place where remote and local information sits. It's a coordinator layer. * NodeState: keeps track of the local status, similar to NodeSyncConnection. * NodeSyncConnection: keeps track of the sending and receiving information and make sure not sending the information the remote node knows. The core protocol is that each node will send {what it has} - {what the target has} to the target. For example, think about node A <-> B. A will send all A has exclude what B has to B. Whenever when there is new information (from NodeState or NodeSyncConnection), it'll be passed to RaySyncer broadcast message to broadcast. NodeSyncConnection is for the communication layer. It has two implementations Client and Server: * Server => Client: client will send a long-polling request and server will response every 100ms if there is data to be sent. * Client => Server: client will check every 100ms to see whether there is new data to be sent. If there is, just use RPC call to send the data. Here is one example: ```mermaid flowchart LR; A-->B; B-->C; B-->D; ``` It means A initialize the connection to B and B initialize the connections to C and D Now C generate a message M: 1. [C] RaySyncer check whether there is new message generated in C and get M 2. [C] RaySyncer will push M to NodeSyncConnection in local component (B) 3. [C] ServerSyncConnection will wait until B send a long polling and send the data to B 4. [B] B received the message from C and push it to local sync connection (C, A, D) 5. [B] ClientSyncConnection of C will not push it to its local queue since it's received by this channel. 6. [B] ClientSyncConnection of D will send this message to D 7. [B] ServerSyncConnection of A will be used to send this message to A (long-polling here) 8. [B] B will update NodeState (local component) with this message M 9. [D] D's pipelines is similar to 5) (with ServerSyncConnection) and 8) 10. [A] A's pipeline is similar to 5) and 8)
2022-03-29 23:52:39 -07:00
cc_binary(
name = "syncer_service_e2e_test",
srcs = ["src/ray/common/test/syncer_service_e2e_test.cc"],
copts = COPTS,
deps = [
":ray_common",
],
)
cc_test(
name = "ray_syncer_test",
srcs = ["src/ray/common/test/ray_syncer_test.cc"],
copts = COPTS,
tags = ["team:core"],
deps = [
":ray_common",
":ray_mock",
"@com_google_googletest//:gtest",
],
)
cc_test(
name = "task_spec_test",
srcs = ["src/ray/common/test/task_spec_test.cc"],
copts = COPTS,
tags = ["team:core"],
deps = [
":ray_common",
"@com_google_googletest//:gtest",
],
)
2021-09-24 20:11:43 -07:00
cc_test(
name = "callback_reply_test",
size = "small",
srcs = ["src/ray/gcs/test/callback_reply_test.cc"],
copts = COPTS,
tags = ["team:core"],
deps = [
":gcs",
"@com_google_googletest//:gtest_main",
],
)
2019-01-20 12:16:47 -08:00
cc_test(
2019-01-20 12:16:47 -08:00
name = "asio_test",
size = "small",
srcs = ["src/ray/gcs/test/asio_test.cc"],
args = [
"$(location redis-server)",
"$(location redis-cli)",
],
copts = COPTS,
data = [
"//:redis-cli",
"//:redis-server",
],
2021-08-18 18:38:31 -07:00
tags = ["team:core"],
2019-01-20 12:16:47 -08:00
deps = [
":gcs",
":ray_util",
"@com_google_googletest//:gtest_main",
2019-01-20 12:16:47 -08:00
],
)
flatbuffer_cc_library(
name = "node_manager_fbs",
srcs = ["src/ray/raylet/format/node_manager.fbs"],
flatc_args = FLATC_ARGS,
2020-07-19 12:53:08 -07:00
out_prefix = "ray/raylet/format/",
)
pyx_library(
name = "_raylet",
srcs = glob([
"python/ray/__init__.py",
"python/ray/_raylet.pxd",
"python/ray/_raylet.pyx",
"python/ray/includes/*.pxd",
"python/ray/includes/*.pxi",
]),
# Export ray ABI symbols, which can then be used by _streaming.so.
# We need to dlopen this lib with RTLD_GLOBAL to use ABI in this
# shared lib, see python/ray/__init__.py.
cc_kwargs = dict(
srcs = PYX_SRCS,
copts = COPTS + PYX_COPTS,
# see https://github.com/tensorflow/tensorflow/blob/r2.1/tensorflow/lite/BUILD#L444
linkopts = select({
"@bazel_tools//src/conditions:darwin": [
"-Wl,-exported_symbols_list,$(location //:src/ray/ray_exported_symbols.lds)",
],
"@bazel_tools//src/conditions:windows": [
],
"//conditions:default": [
"-Wl,--version-script,$(location //:src/ray/ray_version_script.lds)",
],
}),
linkstatic = 1,
),
deps = [
"//:core_worker_lib",
"//:exported_internal",
"//:global_state_accessor_lib",
"//:ray_util",
"//:raylet_lib",
"//:src/ray/ray_exported_symbols.lds",
"//:src/ray/ray_version_script.lds",
"//:stats_lib",
"//src/ray/protobuf:serialization_cc_proto",
],
)
cc_binary(
name = "libcore_worker_library_java.so",
srcs = glob([
"src/ray/core_worker/lib/java/*.h",
"src/ray/core_worker/lib/java/*.cc",
2020-07-19 12:53:08 -07:00
]),
copts = COPTS,
# Export ray ABI symbols, which can then be used by libstreaming_java.so. see `//:_raylet`
linkopts = select({
"@bazel_tools//src/conditions:darwin": [
"-Wl,-exported_symbols_list,$(location //:src/ray/ray_exported_symbols.lds)",
],
"@bazel_tools//src/conditions:windows": [
],
"//conditions:default": [
"-Wl,--version-script,$(location //:src/ray/ray_version_script.lds)",
],
}),
linkshared = 1,
linkstatic = 1,
visibility = ["//java:__subpackages__"],
deps = [
"//:core_worker_lib",
"//:exported_internal",
2020-05-27 17:43:47 +08:00
"//:global_state_accessor_lib",
"//:src/ray/ray_exported_symbols.lds",
"//:src/ray/ray_version_script.lds",
2020-07-22 10:35:08 +08:00
"//:stats_lib",
2020-07-19 12:53:08 -07:00
"@bazel_tools//tools/jdk:jni",
],
)
2019-01-27 18:32:04 -08:00
filegroup(
name = "python_sources",
srcs = glob([
"python/ray/*.py",
"python/ray/autoscaler/*.py",
"python/ray/autoscaler/_private/*.py",
"python/ray/autoscaler/_private/_azure/*.json",
"python/ray/autoscaler/aws/defaults.yaml",
"python/ray/autoscaler/azure/defaults.yaml",
"python/ray/autoscaler/gcp/defaults.yaml",
"python/ray/autoscaler/local/defaults.yaml",
2019-01-27 18:32:04 -08:00
"python/ray/cloudpickle/*.py",
"python/ray/core/__init__.py",
"python/ray/core/generated/__init__.py",
"python/ray/core/generated/ray/__init__.py",
"python/ray/core/generated/ray/protocol/__init__.py",
"python/ray/dashboard/**/*.py",
2019-01-27 18:32:04 -08:00
"python/ray/experimental/*.py",
"python/ray/util/*.py",
2019-01-27 18:32:04 -08:00
"python/ray/internal/*.py",
"python/ray/workers/default_worker.py",
]),
2019-01-27 18:32:04 -08:00
)
alias(
name = "redis-server",
actual = select({
"@bazel_tools//src/conditions:windows": "@com_github_tporadowski_redis_bin//:redis-server.exe",
"//conditions:default": "@com_github_antirez_redis//:redis-server",
2020-02-28 10:18:56 -08:00
}),
visibility = ["//visibility:public"],
)
alias(
name = "redis-cli",
actual = select({
"@bazel_tools//src/conditions:windows": "@com_github_tporadowski_redis_bin//:redis-cli.exe",
"//conditions:default": "@com_github_antirez_redis//:redis-cli",
2020-01-05 21:21:17 -08:00
}),
visibility = ["//visibility:public"],
2019-01-27 18:32:04 -08:00
)
filegroup(
name = "all_py_proto",
srcs = [
"//src/ray/protobuf:agent_manager_py_proto",
"//src/ray/protobuf:common_py_proto",
"//src/ray/protobuf:core_worker_py_proto",
"//src/ray/protobuf:event_py_proto",
"//src/ray/protobuf:gcs_py_proto",
"//src/ray/protobuf:gcs_service_py_proto",
"//src/ray/protobuf:job_agent_py_proto",
"//src/ray/protobuf:node_manager_py_proto",
2020-11-05 16:23:54 -08:00
"//src/ray/protobuf:ray_client_py_proto",
"//src/ray/protobuf:reporter_py_proto",
"//src/ray/protobuf:runtime_env_agent_py_proto",
"//src/ray/protobuf:runtime_env_common_py_proto",
],
)
filegroup(
name = "serve_py_proto",
srcs = [
"//src/ray/protobuf:serve_py_proto",
],
)
# This is a dummy test dependency that causes the python tests to be
# re-run if any of these files changes.
py_library(
name = "ray_lib",
srcs = glob(
["python/ray/**/*.py"],
exclude = ["python/ray/tests/*.py"],
),
visibility = ["__subpackages__"],
)
copy_to_workspace(
name = "cp_raylet_so",
srcs = ["python/ray/_raylet.so"],
dstdir = "python/ray",
)
copy_to_workspace(
name = "cp_all_py_proto",
srcs = [":all_py_proto"],
dstdir = "python/ray/core/generated",
)
copy_to_workspace(
name = "cp_serve_py_proto",
srcs = [":serve_py_proto"],
dstdir = "python/ray/serve/generated",
)
copy_to_workspace(
name = "cp_redis",
srcs = [
":redis-cli",
":redis-server",
],
dstdir = "python/ray/core/src/ray/thirdparty/redis/src",
)
copy_to_workspace(
name = "cp_raylet",
srcs = [":raylet"],
dstdir = "python/ray/core/src/ray/raylet",
)
copy_to_workspace(
name = "cp_gcs_server",
srcs = [":gcs_server"],
dstdir = "python/ray/core/src/ray/gcs",
)
genrule(
name = "install_py_proto",
srcs = [
":cp_all_py_proto",
":cp_serve_py_proto",
],
outs = ["install_py_proto.out"],
cmd = """
# NOTE(hchen): Protobuf doesn't allow specifying Python package name. So we use this `sed`
# command to change the import path in the generated file.
# shellcheck disable=SC2006
files=(`ls python/ray/core/generated/*_pb2*.py` `ls python/ray/serve/generated/*_pb2*.py`)
sed -i -E 's/from src.ray.protobuf/from ./' "$${files[@]}"
sed -i -E 's/from opencensus.proto.metrics.v1 import/from . import/' "$${files[@]}"
sed -i -E 's/from opencensus.proto.resource.v1 import/from . import/' "$${files[@]}"
echo "$${PWD}" > $@
""",
local = 1,
)
2019-01-27 18:32:04 -08:00
genrule(
name = "ray_pkg",
srcs = [
":cp_raylet_so",
":python_sources",
":install_py_proto",
":cp_redis",
":cp_raylet",
":cp_gcs_server",
],
outs = ["ray_pkg.out"],
2019-01-27 18:32:04 -08:00
cmd = """
2020-02-27 15:14:10 -08:00
if [ "$${OSTYPE-}" = "msys" ]; then
2020-07-11 05:27:56 -07:00
ln -P -f -- python/ray/_raylet.so python/ray/_raylet.pyd
fi
echo "$${PWD}" > $@
""",
local = 1,
2019-01-27 18:32:04 -08:00
)