[Release] Release 1.4.0 stress tests, scalability envelope, and microbenchmark release logs (#16228)

release/release_logs/1.4.0/microbenchmark.txt

@@ -0,0 +1,28 @@
+single client get calls per second 44402.31 +- 696.92
+single client put calls per second 31082.75 +- 272.3
+multi client put calls per second 142971.58 +- 973.63
+single client get calls (Plasma Store) per second 9287.73 +- 75.27
+single client put calls (Plasma Store) per second 5591.63 +- 32.73
+multi client put calls (Plasma Store) per second 9351.66 +- 162.42
+single client put gigabytes per second 15.32 +- 10.81
+multi client put gigabytes per second 37.54 +- 1.54
+single client tasks sync per second 1751.39 +- 39.02
+single client tasks async per second 12587.25 +- 480.72
+multi client tasks async per second 42626.82 +- 1036.23
+1:1 actor calls sync per second 2881.46 +- 14.85
+1:1 actor calls async per second 7523.22 +- 79.34
+1:1 actor calls concurrent per second 6307.02 +- 182.72
+1:n actor calls async per second 16096.56 +- 1450.93
+n:n actor calls async per second 49367.23 +- 1282.04
+n:n actor calls with arg async per second 12757.09 +- 106.59
+1:1 async-actor calls sync per second 1952.03 +- 78.92
+1:1 async-actor calls async per second 4001.92 +- 96.38
+1:1 async-actor calls with args async per second 2568.53 +- 64.5
+1:n async-actor calls async per second 16600.97 +- 340.0
+n:n async-actor calls async per second 41847.43 +- 854.91
+client: get calls per second 2032.55 +- 19.19
+client: put calls per second 960.54 +- 17.03
+client: remote put calls per second 57490.74 +- 1047.55
+client: 1:1 actor calls sync per second 584.88 +- 31.88
+client: 1:1 actor calls async per second 663.21 +- 4.98
+client: 1:1 actor calls concurrent per second 666.91 +- 2.77

release/release_logs/1.4.0/scalability/distributed.txt

@@ -0,0 +1,4 @@
+Actor time: 258.023392103 (10000 actors)
+Task time: 619.2078768060001 (10000 tasks)
+PG time: 24.422072359999902 (1000 placement groups)
+Node launch time: 797.4376067439998 (250 nodes)

release/release_logs/1.4.0/scalability/object_store.txt

@@ -0,0 +1,4 @@
+(autoscaler +2m20s) Resized to 212 CPUs.
+Ensure all actors have started.: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 50/50 [00:02<00:00, 22.15it/s]
+Broadcasting objects: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 50/50 [00:00<00:00, 10586.33it/s]
+Broadcast time: 308.624902904 (1073741824 B x 50 nodes)

release/release_logs/1.4.0/scalability/single_node.txt

@@ -0,0 +1,5 @@
+Many args time: 13.782774517000007 (10000 args)
+Many returns time: 5.161780563999997 (3000 returns)
+Ray.get time: 26.583768171000003 (10000 args)
+Queued task time: 145.18449043200002 (1000000 tasks)
+Ray.get large object time: 35.661908608999966 (107374182400 bytes)

release/release_logs/1.4.0/stress_tests/dead_actors.txt

@@ -0,0 +1,4 @@
+Finished in: 200.69372820854187s
+Average iteration time: 2.006935486793518s
+Max iteration time: 41.49158835411072s
+Min iteration time: 0.10450339317321777s

release/release_logs/1.4.0/stress_tests/many_tasks.txt

@@ -0,0 +1,17 @@
+Stage 0 results:
+        Total time: 36.21135091781616
+Stage 1 results:
+        Total time: 210.65460968017578
+        Average iteration time: 21.065443801879884
+        Max iteration time: 27.833787441253662
+        Min iteration time: 18.884577751159668
+Stage 2 results:
+        Total time: 270.4833490848541
+        Average iteration time: 54.0964364528656
+        Max iteration time: 55.11329627037048
+        Min iteration time: 52.94339346885681
+Stage 3 results:
+        Actor creation time: 0.5002284049987793
+        Total time: 2355.353306055069
+Stage 4 results:
+        Scheduling spread: 133.24447738480004.

release/stress_tests/cluster.yaml

@@ -31,13 +31,18 @@ available_node_types:
         min_workers: 100
         max_workers: 100
 
+docker:
+    image: anyscale/ray:nightly
+    container_name: ray_container
+    pull_before_run: False
+
 head_node_type: cpu_64_ondemand
 worker_default_node_type: cpu_2_spot
 
 setup_commands:
     - pip install -U pip
     - pip install terminado
-    - pip install boto3==1.4.8 cython==0.29.0
+    - pip install boto3 cython==0.29.0
     - pip install -U {{env["RAY_WHEEL"]}}
 
 head_start_ray_commands: