[Core] update protobuf to 3.19.4 (#25648)

The error message in #25638 indicates we should use protobuf>3.19.0 to generated code so that we can work with python protobuf >= 4.21.1. Try generating wheels to see if this works.

bazel/ray_deps_setup.bzl

@@ -88,9 +88,9 @@ def ray_deps_setup():
     # https://github.com/ray-project/ray/issues/14117
     http_archive(
         name = "com_google_protobuf",
-        strip_prefix = "protobuf-3.16.0",
-        urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.16.0.tar.gz"],
-        sha256 = "7892a35d979304a404400a101c46ce90e85ec9e2a766a86041bb361f626247f5",
+        strip_prefix = "protobuf-3.19.4",
+        urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.19.4.tar.gz"],
+        sha256 = "3bd7828aa5af4b13b99c191e8b1e884ebfa9ad371b0ce264605d347f135d2568",
     )
 
     # NOTE(lingxuan.zlx): 3rd party dependencies could be accessed, so it suggests

python/ray/tests/BUILD

@@ -60,6 +60,7 @@ py_test_module_list(
     "test_healthcheck.py",
     "test_kill_raylet_signal_log.py",
     "test_memstat.py",
+    "test_protobuf_compatibility.py"
   ],
   size = "medium",
   extra_srcs = SRCS,

python/ray/tests/test_protobuf_compatibility.py

@@ -0,0 +1,35 @@
+import sys
+
+import pytest
+
+import ray
+
+
+@pytest.mark.skipif(
+    sys.platform == "win32",
+    reason="Failing on Windows, affected by protocolbuffers/protobuf#10075.",
+)
+def test_protobuf_compatibility(shutdown_only):
+    protobuf_4_21_0 = {"pip": ["protobuf==4.21.0"]}
+    protobuf_3_12_2 = {"pip": ["protobuf==3.12.2"]}
+
+    ray.init()
+
+    @ray.remote
+    def load_ray():
+        import google.protobuf
+
+        # verfiy this no longer crashes. see
+        # https://github.com/ray-project/ray/issues/25282
+        import ray  # noqa
+
+        return google.protobuf.__version__
+
+    assert "4.21.0" == ray.get(load_ray.options(runtime_env=protobuf_4_21_0).remote())
+    assert "3.12.2" == ray.get(load_ray.options(runtime_env=protobuf_3_12_2).remote())
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-sv", __file__]))

python/requirements.txt

@@ -18,7 +18,7 @@ msgpack >= 1.0.0, < 2.0.0
 numpy >= 1.16
 opencensus
 prometheus_client >= 0.7.1, < 0.14.0
-protobuf >= 3.8.0, < 4.0.0
+protobuf >= 3.15.3, < 4.0.0
 py-spy >= 0.2.0
 pydantic >= 1.8
 pyyaml

release/requirements.txt

@@ -11,7 +11,7 @@ toml
 python-dotenv
 expiringdict
 requests
-protobuf<4
+protobuf >= 3.15.3, < 4.0.0
 pytz
 git+https://github.com/ray-project/xgboost_ray.git#egg=xgboost_ray
 git+https://github.com/ray-project/lightgbm_ray.git#lightgbm_ray

release/requirements_buildkite.txt

@@ -2,7 +2,7 @@ anyscale
 click
 boto3
 jinja2
-protobuf<4
+protobuf >= 3.15.3, < 4.0.0
 pydantic
 pyyaml
 requests