mirror of
https://github.com/vale981/ray
synced 2025-03-06 02:21:39 -05:00
Release 0.8.0 test logs (#6512)
This commit is contained in:
Eric Liang
GitHub
parent
ab5a9f0946
commit
6725a61bda
7 changed files with 77 additions and 22 deletions
|
|
@ -47,6 +47,20 @@ class Actor(object):
|
|||
return np.ones(size, dtype=np.uint8)
|
||||
|
||||
|
||||
# Stage 0: Submit a bunch of small tasks with large returns.
|
||||
stage_0_iterations = []
|
||||
start_time = time.time()
|
||||
logger.info("Submitting many tasks with large returns.")
|
||||
for i in range(10):
|
||||
iteration_start = time.time()
|
||||
logger.info("Iteration %s", i)
|
||||
ray.get([f.remote(1000000) for _ in range(1000)])
|
||||
stage_0_iterations.append(time.time() - iteration_start)
|
||||
|
||||
stage_0_time = time.time() - start_time
|
||||
logger.info("Finished stage 0 after %s seconds.", stage_0_time)
|
||||
|
||||
|
||||
# Stage 1: Launch a bunch of tasks.
|
||||
stage_1_iterations = []
|
||||
start_time = time.time()
|
||||
|
|
@ -101,6 +115,9 @@ for N in [1000, 100000]:
|
|||
stage_3_time = time.time() - start_time
|
||||
logger.info("Finished stage 3 in %s seconds.", stage_3_time)
|
||||
|
||||
print("Stage 0 results:")
|
||||
print("\tTotal time: {}".format(stage_0_time))
|
||||
|
||||
print("Stage 1 results:")
|
||||
print("\tTotal time: {}".format(stage_1_time))
|
||||
print("\tAverage iteration time: {}".format(
|
||||
|
|
|
|||
|
|
@ -64,10 +64,9 @@ This document describes the process for creating new releases.
|
|||
ray microbenchmark
|
||||
|
||||
Run `ray microbenchmark` on an `m4.16xl` instance running `Ubuntu 18.04` with `Python 3` to get the
|
||||
latest microbenchmark numbers and update them in `profiling.rst`.
|
||||
latest microbenchmark numbers.
|
||||
|
||||
The results should be updated in ``doc/dev/profiling.rst`` and checked in under
|
||||
``doc/dev/release_logs/<version>``.
|
||||
The results should be checked in under ``doc/dev/release_logs/<version>``.
|
||||
|
||||
5. **Resolve release-blockers:** If a release blocking issue arises, there are
|
||||
two ways the issue can be resolved: 1) Fix the issue on the master branch and
|
||||
|
|
|
|||
14
doc/dev/release_logs/0.8.0/microbenchmark.txt
Normal file
14
doc/dev/release_logs/0.8.0/microbenchmark.txt
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
single client get calls per second 30526.52 +- 1121.22
|
||||
single client put calls per second 6428.11 +- 51.25
|
||||
single client put gigabytes per second 12.85 +- 5.45
|
||||
multi client put calls per second 16976.5 +- 60.13
|
||||
multi client put gigabytes per second 22.54 +- 0.86
|
||||
single client tasks sync per second 1223.63 +- 18.36
|
||||
single client tasks async per second 13715.93 +- 229.68
|
||||
multi client tasks async per second 38344.2 +- 4336.41
|
||||
1:1 actor calls sync per second 2318.72 +- 24.19
|
||||
1:1 actor calls async per second 7530.01 +- 222.72
|
||||
1:1 actor calls concurrent per second 6981.48 +- 228.29
|
||||
1:n actor calls async per second 13974.54 +- 41.26
|
||||
n:n actor calls async per second 43835.4 +- 400.04
|
||||
n:n actor calls with arg async per second 14775.69 +- 295.24
|
||||
24
doc/dev/release_logs/0.8.0/rllib_regression.txt
Normal file
24
doc/dev/release_logs/0.8.0/rllib_regression.txt
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
+----------------------------------------+------------+--------------------+--------+------------------+-------------+----------+
|
||||
| Trial name | status | loc | iter | total time (s) | timesteps | reward |
|
||||
|----------------------------------------+------------+--------------------+--------+------------------+-------------+----------|
|
||||
| IMPALA_BreakoutNoFrameskip-v4_e25c5ce8 | TERMINATED | | 310 | 3604.3 | 9648000 | 399.75 |
|
||||
| IMPALA_BreakoutNoFrameskip-v4_e25cdfec | TERMINATED | | 310 | 3611.22 | 9552000 | 403.49 |
|
||||
| IMPALA_BreakoutNoFrameskip-v4_e25d65de | TERMINATED | | 310 | 3604.05 | 9604500 | 390.47 |
|
||||
| IMPALA_BreakoutNoFrameskip-v4_e25ddf0a | TERMINATED | | 310 | 3604.18 | 9447000 | 365.57 |
|
||||
| PPO_BreakoutNoFrameskip-v4_e25e4ddc | TERMINATED | | 604 | 3605.24 | 3020000 | 42.59 |
|
||||
| PPO_BreakoutNoFrameskip-v4_e25eb8da | TERMINATED | | 613 | 3603.35 | 3065000 | 50.58 |
|
||||
| PPO_BreakoutNoFrameskip-v4_e25f4520 | TERMINATED | | 608 | 3600.01 | 3040000 | 51.2 |
|
||||
| PPO_BreakoutNoFrameskip-v4_e25fd44a | TERMINATED | | 613 | 3604.97 | 3065000 | 26.19 |
|
||||
| APEX_BreakoutNoFrameskip-v4_e2603fa2 | TERMINATED | | 115 | 3622.22 | 5169280 | 48.56 |
|
||||
| APEX_BreakoutNoFrameskip-v4_e260d462 | TERMINATED | | 115 | 3616.47 | 5201280 | 45.01 |
|
||||
| APEX_BreakoutNoFrameskip-v4_e2615a90 | TERMINATED | | 115 | 3615.15 | 5134080 | 25.02 |
|
||||
| APEX_BreakoutNoFrameskip-v4_e261cbec | TERMINATED | | 115 | 3625.55 | 5189280 | 47.01 |
|
||||
| A2C_BreakoutNoFrameskip-v4_e26260a2 | TERMINATED | | 351 | 3602.43 | 3478000 | 145.84 |
|
||||
| A2C_BreakoutNoFrameskip-v4_e26331f8 | TERMINATED | | 351 | 3601.93 | 3577000 | 178.73 |
|
||||
| A2C_BreakoutNoFrameskip-v4_e2639e68 | TERMINATED | | 351 | 3607.28 | 3521500 | 185.45 |
|
||||
| A2C_BreakoutNoFrameskip-v4_e2642630 | TERMINATED | | 351 | 3606.35 | 3460000 | 122.32 |
|
||||
| DQN_BreakoutNoFrameskip-v4_99c588f2 | TERMINATED | | 32 | 3643.26 | 320000 | 19.24 |
|
||||
| DQN_BreakoutNoFrameskip-v4_99c62de8 | TERMINATED | | 31 | 3619.43 | 310000 | 19.97 |
|
||||
| DQN_BreakoutNoFrameskip-v4_99c6c01e | TERMINATED | | 33 | 3680.95 | 330000 | 16.91 |
|
||||
| DQN_BreakoutNoFrameskip-v4_99c73f44 | TERMINATED | | 31 | 3625.12 | 310000 | 16.23 |
|
||||
+----------------------------------------+------------+--------------------+--------+------------------+-------------+----------+
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
Finished in: 220.28446006774902s
|
||||
Average iteration time: 2.2028422021865843s
|
||||
Max iteration time: 4.1706767082214355s
|
||||
Min iteration time: 1.1767382621765137s
|
||||
15
doc/dev/release_logs/0.8.0/stress_tests/test_many_tasks.txt
Normal file
15
doc/dev/release_logs/0.8.0/stress_tests/test_many_tasks.txt
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
Stage 0 results:
|
||||
Total time: 18.916658878326416
|
||||
Stage 1 results:
|
||||
Total time: 112.98085951805115
|
||||
Average iteration time: 11.298071336746215
|
||||
Max iteration time: 11.726166009902954
|
||||
Min iteration time: 11.037099123001099
|
||||
Stage 2 results:
|
||||
Total time: 1492.7644929885864
|
||||
Average iteration time: 298.5524812221527
|
||||
Max iteration time: 334.69453477859497
|
||||
Min iteration time: 233.40272068977356
|
||||
Stage 3 results:
|
||||
Actor creation time: 0.03908824920654297
|
||||
Total time: 2432.014177799225
|
||||
|
|
@ -61,25 +61,7 @@ To run a set of single-node Ray microbenchmarks, use:
|
|||
|
||||
ray microbenchmark
|
||||
|
||||
The following are the results for the 0.7.6 release on a m4.16xl instance running
|
||||
Ubuntu 18.04 and Python 3.6:
|
||||
|
||||
.. code-block:: text
|
||||
|
||||
single client get calls per second 28595.02 +- 580.33
|
||||
single client put calls per second 6313.62 +- 66.88
|
||||
single client put gigabytes per second 11.6 +- 6.86
|
||||
multi client put calls per second 16800.89 +- 381.69
|
||||
multi client put gigabytes per second 23.33 +- 0.96
|
||||
single client tasks sync per second 1963.72 +- 48.48
|
||||
single client tasks async per second 5181.29 +- 30.0
|
||||
multi client tasks async per second 5566.7 +- 280.72
|
||||
1:1 actor calls sync per second 1595.47 +- 38.32
|
||||
1:1 actor calls async per second 2496.26 +- 37.62
|
||||
1:1 direct actor calls async per second 7233.63 +- 205.75
|
||||
n:n actor calls async per second 5357.63 +- 116.9
|
||||
n:n direct actor calls async per second 90703.32 +- 805.56
|
||||
n:n direct actor calls with arg async per second 13300.47 +- 532.66
|
||||
You can find the microbenchmark results for Ray releases in the `GitHub release logs <https://github.com/ray-project/ray/tree/master/doc/dev/release_logs>`__.
|
||||
|
||||
References
|
||||
----------
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue