hiro/ray
1
0
Fork 0
mirror of https://github.com/vale981/ray synced 2025-03-07 02:51:39 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

[SGD] Provide better error message if model graphs have different numbers of variables (#3139)

Browse source
This commit is contained in:
Philipp Moritz 2018-10-25 22:18:10 -07:00 • committed by Robert Nishihara
parent d34516f1f8
commit d3148cc3ab
1 changed files with 9 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
python/ray/experimental/sgd/modified_allreduce.py
View file

@ -584,7 +584,15 @@ def pack_small_tensors(tower_grads, max_bytes=0):
if len(small_ranges):
new_tower_grads = []
for dev_idx, gv_list in enumerate(tower_grads):
assert len(gv_list) == num_gv
assert len(gv_list) == num_gv, (
"Possible cause: "
"Networks constructed on different workers "
"don't have the same number of variables. "
"If you use tf.GraphKeys or tf.global_variables() "
"with multiple graphs per worker during network "
"construction, you need to use "
"appropriate scopes, see "
"https://github.com/ray-project/ray/issues/3136")
new_gv_list = []
for r in small_ranges:
key = '%d:%d' % (dev_idx, len(new_gv_list))