diff --git a/python/ray/rllib/optimizers/segment_tree.py b/python/ray/rllib/optimizers/segment_tree.py
index 3409aa667..b412a89bd 100644
--- a/python/ray/rllib/optimizers/segment_tree.py
+++ b/python/ray/rllib/optimizers/segment_tree.py
@@ -78,10 +78,9 @@ class SegmentTree(object):
           elements.
         """
         if end is None:
-            end = self._capacity
+            end = self._capacity - 1
         if end < 0:
             end += self._capacity
-        end -= 1
         return self._reduce_helper(start, end, 1, 0, self._capacity - 1)
 
     def __setitem__(self, idx, val):