From 97582a802d4519ae72120464bf9c1c6333eec4bc Mon Sep 17 00:00:00 2001
From: Chen Shen <scv119@gmail.com>
Date: Sat, 18 Jun 2022 16:06:56 -0700
Subject: [PATCH] [Core] update protobuf to 3.19.4 (#25648)

The error message in #25638 indicates we should use protobuf>3.19.0 to generated code so that we can work with python protobuf >= 4.21.1. Try generating wheels to see if this works.
---
 bazel/ray_deps_setup.bzl                      |  6 ++--
 python/ray/tests/BUILD                        |  1 +
 .../ray/tests/test_protobuf_compatibility.py  | 35 +++++++++++++++++++
 python/requirements.txt                       |  2 +-
 release/requirements.txt                      |  2 +-
 release/requirements_buildkite.txt            |  2 +-
 6 files changed, 42 insertions(+), 6 deletions(-)
 create mode 100644 python/ray/tests/test_protobuf_compatibility.py

diff --git a/bazel/ray_deps_setup.bzl b/bazel/ray_deps_setup.bzl
index 144c9121a..faa151ab6 100644
--- a/bazel/ray_deps_setup.bzl
+++ b/bazel/ray_deps_setup.bzl
@@ -88,9 +88,9 @@ def ray_deps_setup():
     # https://github.com/ray-project/ray/issues/14117
     http_archive(
         name = "com_google_protobuf",
-        strip_prefix = "protobuf-3.16.0",
-        urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.16.0.tar.gz"],
-        sha256 = "7892a35d979304a404400a101c46ce90e85ec9e2a766a86041bb361f626247f5",
+        strip_prefix = "protobuf-3.19.4",
+        urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.19.4.tar.gz"],
+        sha256 = "3bd7828aa5af4b13b99c191e8b1e884ebfa9ad371b0ce264605d347f135d2568",
     )
 
     # NOTE(lingxuan.zlx): 3rd party dependencies could be accessed, so it suggests
diff --git a/python/ray/tests/BUILD b/python/ray/tests/BUILD
index 1a8286330..91b3458fa 100644
--- a/python/ray/tests/BUILD
+++ b/python/ray/tests/BUILD
@@ -60,6 +60,7 @@ py_test_module_list(
     "test_healthcheck.py",
     "test_kill_raylet_signal_log.py",
     "test_memstat.py",
+    "test_protobuf_compatibility.py"
   ],
   size = "medium",
   extra_srcs = SRCS,
diff --git a/python/ray/tests/test_protobuf_compatibility.py b/python/ray/tests/test_protobuf_compatibility.py
new file mode 100644
index 000000000..847a886ad
--- /dev/null
+++ b/python/ray/tests/test_protobuf_compatibility.py
@@ -0,0 +1,35 @@
+import sys
+
+import pytest
+
+import ray
+
+
+@pytest.mark.skipif(
+    sys.platform == "win32",
+    reason="Failing on Windows, affected by protocolbuffers/protobuf#10075.",
+)
+def test_protobuf_compatibility(shutdown_only):
+    protobuf_4_21_0 = {"pip": ["protobuf==4.21.0"]}
+    protobuf_3_12_2 = {"pip": ["protobuf==3.12.2"]}
+
+    ray.init()
+
+    @ray.remote
+    def load_ray():
+        import google.protobuf
+
+        # verfiy this no longer crashes. see
+        # https://github.com/ray-project/ray/issues/25282
+        import ray  # noqa
+
+        return google.protobuf.__version__
+
+    assert "4.21.0" == ray.get(load_ray.options(runtime_env=protobuf_4_21_0).remote())
+    assert "3.12.2" == ray.get(load_ray.options(runtime_env=protobuf_3_12_2).remote())
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-sv", __file__]))
diff --git a/python/requirements.txt b/python/requirements.txt
index 78edf79d9..fcf4c9661 100644
--- a/python/requirements.txt
+++ b/python/requirements.txt
@@ -18,7 +18,7 @@ msgpack >= 1.0.0, < 2.0.0
 numpy >= 1.16
 opencensus
 prometheus_client >= 0.7.1, < 0.14.0
-protobuf >= 3.8.0, < 4.0.0
+protobuf >= 3.15.3, < 4.0.0
 py-spy >= 0.2.0
 pydantic >= 1.8
 pyyaml
diff --git a/release/requirements.txt b/release/requirements.txt
index 5fc5e642a..974eabf2c 100644
--- a/release/requirements.txt
+++ b/release/requirements.txt
@@ -11,7 +11,7 @@ toml
 python-dotenv
 expiringdict
 requests
-protobuf<4
+protobuf >= 3.15.3, < 4.0.0
 pytz
 git+https://github.com/ray-project/xgboost_ray.git#egg=xgboost_ray
 git+https://github.com/ray-project/lightgbm_ray.git#lightgbm_ray
\ No newline at end of file
diff --git a/release/requirements_buildkite.txt b/release/requirements_buildkite.txt
index cb06e2616..a563ac281 100644
--- a/release/requirements_buildkite.txt
+++ b/release/requirements_buildkite.txt
@@ -2,7 +2,7 @@ anyscale
 click
 boto3
 jinja2
-protobuf<4
+protobuf >= 3.15.3, < 4.0.0
 pydantic
 pyyaml
 requests