From 7a24144bfd6202e8f8e9aeb11bc3de60ed1d8619 Mon Sep 17 00:00:00 2001 From: mehrdadn Date: Tue, 17 Dec 2019 02:38:36 -0800 Subject: [PATCH] Polish Bazel build scripts (#6424) * Polish Bazel build scripts * Remove glog references from streaming_logging.cc * Move out COPTS and reference them * Disable streaming on Windows * Remove -fno-gnu-unique --- BUILD.bazel | 64 +++++++++---------------- bazel/ray.bzl | 19 +++++--- streaming/BUILD.bazel | 10 ++++ streaming/src/util/streaming_logging.cc | 3 -- 4 files changed, 46 insertions(+), 50 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 42a953ab4..32fd1ecf6 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -7,41 +7,7 @@ load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library") load("@com_github_grpc_grpc//bazel:cython_library.bzl", "pyx_library") load("@rules_proto_grpc//python:defs.bzl", "python_grpc_compile") load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library") -load("//bazel:ray.bzl", "if_linux_x86_64") - -config_setting( - name = "windows", - values = {"cpu": "x64_windows"}, - visibility = ["//visibility:public"], -) - -config_setting( - name = "macos", - values = { - "apple_platform_type": "macos", - "cpu": "darwin", - }, - visibility = ["//visibility:public"], -) - -config_setting( - name = "linux_x86_64", - values = {"cpu": "k8"}, - visibility = ["//visibility:public"], -) - -# TODO(mehrdadn): (How to) support dynamic linking? -PROPAGATED_WINDOWS_DEFINES = ["RAY_STATIC"] - -COPTS = ["-DRAY_USE_GLOG"] + select({ - "@bazel_tools//src/conditions:windows": [ - "-DWIN32_LEAN_AND_MEAN=", # Block the inclusion of WinSock.h, which is obsolete and causes errors - "-Wno-builtin-macro-redefined", # To get rid of warnings caused by deterministic build macros (e.g. #define __DATE__ "redacted") - "-Wno-microsoft-unqualified-friend", # This shouldn't normally be enabled, but otherwise we get: google/protobuf/map_field.h: warning: unqualified friend declaration referring to type outside of the nearest enclosing namespace is a Microsoft extension; add a nested name specifier (for: friend class DynamicMessage) - ] + ["-D" + define for define in PROPAGATED_WINDOWS_DEFINES], - "//conditions:default": [ - ], -}) +load("//bazel:ray.bzl", "COPTS", "PROPAGATED_WINDOWS_DEFINES") # === Begin of protobuf definitions === @@ -938,16 +904,17 @@ pyx_library( "linkstatic": 1, # see https://github.com/tensorflow/tensorflow/blob/r2.1/tensorflow/lite/BUILD#L444 "linkopts": select({ - "//:macos": [ + "@bazel_tools//src/conditions:darwin": [ "-Wl,-exported_symbols_list,$(location //:src/ray/ray_exported_symbols.lds)", ], - "//:windows": [], + "@bazel_tools//src/conditions:windows": [ + ], "//conditions:default": [ "-Wl,--version-script,$(location //:src/ray/ray_version_script.lds)", ], }), }, - copts = COPTS + if_linux_x86_64(["-fno-gnu-unique"]), + copts = COPTS, deps = [ "//:core_worker_lib", "//:raylet_lib", @@ -969,6 +936,7 @@ pyx_library( "python/ray/streaming/includes/*.pxd", "python/ray/streaming/includes/*.pxi", ]), + copts = COPTS, deps = [ "//streaming:streaming_lib", ], @@ -1098,11 +1066,23 @@ py_library( visibility = ["__subpackages__"], ) +genrule( + name = "cp_streaming_lib", + srcs = ["python/ray/streaming/_streaming.so"], + outs = ["cp_streaming_lib.out"], + cmd = """ + set -x && + WORK_DIR=$$(pwd) && + cp -f $(location python/ray/streaming/_streaming.so) "$$WORK_DIR/python/ray/streaming" && + echo "$$WORK_DIR" > $@ + """, + local = 1, +) + genrule( name = "ray_pkg", srcs = [ "python/ray/_raylet.so", - "python/ray/streaming/_streaming.so", "//:python_sources", "//:all_py_proto", "//:redis-server", @@ -1112,13 +1092,15 @@ genrule( "//:raylet_monitor", "@plasma//:plasma_store_server", "//streaming:copy_streaming_py_proto", - ], + ] + select({ + "@bazel_tools//src/conditions:windows": [], # ignore build streaming for windows + "//conditions:default": [":cp_streaming_lib"], + }), outs = ["ray_pkg.out"], cmd = """ set -x && WORK_DIR=$$(pwd) && cp -f $(location python/ray/_raylet.so) "$$WORK_DIR/python/ray" && - cp -f $(location python/ray/streaming/_streaming.so) $$WORK_DIR/python/ray/streaming && mkdir -p "$$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/" && cp -f $(location //:redis-server) "$$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/" && cp -f $(location //:redis-cli) "$$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/" && diff --git a/bazel/ray.bzl b/bazel/ray.bzl index 738a5f0d3..a3ef5dd57 100644 --- a/bazel/ray.bzl +++ b/bazel/ray.bzl @@ -2,6 +2,19 @@ load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_library_publ load("@com_github_checkstyle_java//checkstyle:checkstyle.bzl", "checkstyle_test") load("@bazel_common//tools/maven:pom_file.bzl", "pom_file") +# TODO(mehrdadn): (How to) support dynamic linking? +PROPAGATED_WINDOWS_DEFINES = ["RAY_STATIC"] + +COPTS = ["-DRAY_USE_GLOG"] + select({ + "@bazel_tools//src/conditions:windows": [ + "-DWIN32_LEAN_AND_MEAN=", # Block the inclusion of WinSock.h, which is obsolete and causes errors + "-Wno-builtin-macro-redefined", # To get rid of warnings caused by deterministic build macros (e.g. #define __DATE__ "redacted") + "-Wno-microsoft-unqualified-friend", # This shouldn't normally be enabled, but otherwise we get: google/protobuf/map_field.h: warning: unqualified friend declaration referring to type outside of the nearest enclosing namespace is a Microsoft extension; add a nested name specifier (for: friend class DynamicMessage) + ] + ["-D" + define for define in PROPAGATED_WINDOWS_DEFINES], + "//conditions:default": [ + ], +}) + def flatbuffer_py_library(name, srcs, outs, out_prefix, includes = [], include_paths = []): flatbuffer_library_public( name = name, @@ -64,9 +77,3 @@ def define_java_module( "{auto_gen_header}": "", }, ) - -def if_linux_x86_64(a): - return select({ - "//:linux_x86_64": a, - "//conditions:default": [], - }) diff --git a/streaming/BUILD.bazel b/streaming/BUILD.bazel index 876b54da2..3a6079f4e 100644 --- a/streaming/BUILD.bazel +++ b/streaming/BUILD.bazel @@ -3,6 +3,7 @@ load("@com_github_grpc_grpc//bazel:cython_library.bzl", "pyx_library") load("@rules_proto_grpc//python:defs.bzl", "python_proto_compile") +load("//bazel:ray.bzl", "COPTS") proto_library( name = "streaming_proto", @@ -30,6 +31,7 @@ cc_proto_library( # be linked into streaming libs by dynamic linker. See bazel rule `//:_raylet` cc_binary( name = "ray_util.so", + copts = COPTS, linkshared = 1, visibility = ["//visibility:public"], deps = ["//:ray_util"], @@ -37,6 +39,7 @@ cc_binary( cc_binary( name = "ray_common.so", + copts = COPTS, linkshared = 1, visibility = ["//visibility:public"], deps = ["//:ray_common"], @@ -44,6 +47,7 @@ cc_binary( cc_binary( name = "core_worker_lib.so", + copts = COPTS, linkshared = 1, deps = ["//:core_worker_lib"], ) @@ -56,6 +60,7 @@ cc_library( hdrs = glob([ "src/util/*.h", ]), + copts = COPTS, includes = [ "src", ], @@ -75,6 +80,7 @@ cc_library( hdrs = glob([ "src/config/*.h", ]), + copts = COPTS, deps = [ "ray_common.so", ":streaming_cc_proto", @@ -90,6 +96,7 @@ cc_library( hdrs = glob([ "src/message/*.h", ]), + copts = COPTS, deps = [ "ray_common.so", ":streaming_config", @@ -105,6 +112,7 @@ cc_library( hdrs = glob([ "src/queue/*.h", ]), + copts = COPTS, deps = [ "core_worker_lib.so", "ray_common.so", @@ -128,6 +136,7 @@ cc_library( "src/queue/*.h", "src/test/*.h", ]), + copts = COPTS, includes = ["src"], visibility = ["//visibility:public"], deps = [ @@ -154,6 +163,7 @@ cc_binary( srcs = glob(["src/test/*.h"]) + [ "src/test/mock_actor.cc", ], + copts = COPTS, includes = [ "streaming/src/test", ], diff --git a/streaming/src/util/streaming_logging.cc b/streaming/src/util/streaming_logging.cc index c4c9c6c49..a27c05654 100644 --- a/streaming/src/util/streaming_logging.cc +++ b/streaming/src/util/streaming_logging.cc @@ -2,9 +2,6 @@ #include #include -#include "glog/log_severity.h" -#include "glog/logging.h" - #include "streaming_logging.h" namespace ray {