make progressBar aware of terminal resize, added percentage to progressBarFanc, fixed some tty reservation related issues

jobmanager/progress.py

@@ -93,7 +93,10 @@ class Loop(object):
         self._sigterm = sigterm
         self._name = name
         self._auto_kill_on_last_resort = auto_kill_on_last_resort
-        self._identifier = None
+        
+        if not hasattr(self, '_identifier'):
+            self._identifier = None
+        
  
     def __enter__(self):
         return self
@@ -135,10 +138,14 @@ class Loop(object):
             if self.verbose > 1:
                 print("{}: cleanup successful".format(self._identifier))
             self._proc = None
+            self._identifier = get_identifier(self._name, 'not started')
+        else:
+            raise RuntimeError("{}: cleanup FAILED!".format(self._identifier))
+            
 
     @staticmethod
     def _wrapper_func(func, args, shared_mem_run, shared_mem_pause, interval, verbose, sigint, sigterm, name):
-        """to be executed as a seperate process (that's why this functions is declared static)
+        """to be executed as a separate process (that's why this functions is declared static)
         """
         # implement the process specific signal handler
         identifier = get_identifier(name)
@@ -205,6 +212,7 @@ class Loop(object):
         
         self.__cleanup()
         
+        
     def join(self, timeout):
         """
         calls join for the spawned process with given timeout
@@ -345,6 +353,9 @@ class Progress(Loop):
         
         verbose, sigint, sigterm -> see loop class  
         """
+        self.name = name
+        self._identifier = get_identifier(self.name, pid='not started')
+        
         try:
             for c in count:
                 assert isinstance(c, mp.sharedctypes.Synchronized), "each element of 'count' must be if the type multiprocessing.sharedctypes.Synchronized"
@@ -372,10 +383,7 @@ class Progress(Loop):
         self.start_time = []
         self.speed_calc_cycles = speed_calc_cycles
         
-        if width == 'auto':
-            self.width = get_terminal_width(default=80, name=name, verbose=verbose)
-        else:
-            self.width = width
+        self.width = width
         
         self.q = []
         self.prepend = []
@@ -410,26 +418,12 @@ class Progress(Loop):
         
         self.interval = interval
         self.verbose = verbose
-        self.name = name
-        self.show_on_exit = False
 
+        self.show_on_exit = False
         self.add_args = {}
         
-        # before printing any output to stout, we can now check this
-        # variable to see if any other ProgressBar has reserved that
-        # terminal.
-        func = Progress.show_stat_wrapper_multi
-        if (self.__class__.__name__ in TERMINAL_PRINT_LOOP_CLASSES):
-            self.terminal_reserved = terminal_reserve()
-            if not self.terminal_reserved:
-                if verbose > 1:
-                    warnings.warn("tty reserved, not printing progress!")
-                func = lambda *x: None
-        else:
-            self.terminal_reserved = False
-        
         # setup loop class with func
-        super(Progress, self).__init__(func=func, 
+        super(Progress, self).__init__(func=Progress.show_stat_wrapper_multi, 
                          args=(self.count,
                                self.last_count, 
                                self.start_time,
@@ -452,23 +446,7 @@ class Progress(Loop):
                          auto_kill_on_last_resort=True)
 
     def __exit__(self, *exc_args):
-        """ Tear things down
-        
-        - will terminate loop process
-        - show a last progress -> see the full 100% on exit
-        - releases terminal reservation
-        """
-        super(Progress, self).__exit__(*exc_args)
-        
-        if self.terminal_reserved:
-            if self.show_on_exit:
-                self._show_stat()
-                print('\n'*(self.len-1))
-#             print("reserved __exit__", remove_ESC_SEQ_from_string(self._identifier))
-            terminal_unreserve()
-        else:
-            pass
-#             print("not reserved __exit__", remove_ESC_SEQ_from_string(self._identifier))
+        self.stop()
             
         
     @staticmethod
@@ -669,6 +647,16 @@ class Progress(Loop):
         sys.stdout.flush()
 
     def start(self):
+        # before printing any output to stout, we can now check this
+        # variable to see if any other ProgressBar has reserved that
+        # terminal.
+        
+        if (self.__class__.__name__ in TERMINAL_PRINT_LOOP_CLASSES):
+            if not terminal_reserve(progress_obj=self, verbose=self.verbose, identifier=self._identifier):
+                if self.verbose > 1:
+                    print("{}: tty already reserved, NOT starting the progress loop!".format(self._identifier))
+                return
+        
         super(Progress, self).start()
         self.show_on_exit = True
 
@@ -676,17 +664,20 @@ class Progress(Loop):
         """
             trigger clean up by hand, needs to be done when not using
             context management via 'with' statement
+        
+            - will terminate loop process
+            - show a last progress -> see the full 100% on exit
+            - releases terminal reservation
         """
+        self._auto_kill_on_last_resort = make_sure_its_down 
+            
         super(Progress, self).stop()
-        self._show_stat()
-        print('\n'*(self.len-1))
+        terminal_unreserve(progress_obj=self, verbose=self.verbose, identifier=self._identifier)
+
+        if self.show_on_exit:
+            self._show_stat()
+            print('\n'*(self.len-1))
         self.show_on_exit = False
-        if make_sure_its_down and (self._proc is not None):
-            check_process_termination(proc                     = self._proc, 
-                                      identifier               = self._identifier,
-                                      timeout                  = 2*self.interval, 
-                                      verbose                  = self.verbose, 
-                                      auto_kill_on_last_resort = True)
         
 
 class ProgressBar(Progress):
@@ -731,6 +722,9 @@ class ProgressBar(Progress):
                                                  ESC_BOLD + ESC_GREEN,
                                                  humanize_time(tet), humanize_speed(speed), count_value))             
         else:
+            if width == 'auto':
+                width = get_terminal_width()
+            
             # deduce relative progress and show as bar on screen
             if ttg is None:
                 s3 = "] TTG --"
@@ -846,6 +840,10 @@ class ProgressBarCounter(Progress):
                                      humanize_time(counter_tet),
                                      humanize_speed(counter_speed.value), 
                                      counter_count.value)
+
+        if width == 'auto':
+            width = get_terminal_width()        
+        
         if max_count_value != 0:
             s_c += ' - '
         
@@ -904,41 +902,49 @@ class ProgressBarFancy(Progress):
                          name=name)
         
     @staticmethod        
-    def get_d(s1, s2, width, lp):
-        d = width-len(remove_ESC_SEQ_from_string(s1))-len(remove_ESC_SEQ_from_string(s2))-2-lp
+    def get_d(s1, s2, width, lp, lps):
+        d = width-len(remove_ESC_SEQ_from_string(s1))-len(remove_ESC_SEQ_from_string(s2))-2-lp-lps
         if d >= 0:
-            return s1, s2, d
+            d1 = d // 2
+            d2 = d - d1
+            return s1, s2, d1, d2
 
     @staticmethod
-    def full_stat(p, tet, speed, ttg, eta, ort, repl_ch, width, lp):
-        s1 = "TET {} SPE {:>10} TTG {}".format(tet, speed, ttg)
+    def full_stat(p, tet, speed, ttg, eta, ort, repl_ch, width, lp, lps):
+        s1 = "TET {} SPE {:<10} TTG {}".format(tet, speed, ttg)
         s2 = "ETA {} ORT {}".format(eta, ort)
-        return ProgressBarFancy.get_d(s1, s2, width, lp)        
+        return ProgressBarFancy.get_d(s1, s2, width, lp, lps)
 
 
     @staticmethod
-    def full_minor_stat(p, tet, speed, ttg, eta, ort, repl_ch, width, lp):
-        s1 = "E {} S {:>10} G {}".format(tet, speed, ttg)
-        s2 = "A {} O{}".format(eta, ort)
-        return ProgressBarFancy.get_d(s1, s2, width, lp)
+    def full_minor_stat(p, tet, speed, ttg, eta, ort, repl_ch, width, lp, lps):
+        s1 = "E {} S {:<10} G {}".format(tet, speed, ttg)
+        s2 = "A {} O {}".format(eta, ort)
+        return ProgressBarFancy.get_d(s1, s2, width, lp, lps)
 
     @staticmethod
-    def reduced_1_stat(p, tet, speed, ttg, eta, ort, repl_ch, width, lp):
-        s1 = "E {} S {:>10} G {}".format(tet, speed, ttg)
+    def reduced_1_stat(p, tet, speed, ttg, eta, ort, repl_ch, width, lp, lps):
+        s1 = "E {} S {:<10} G {}".format(tet, speed, ttg)
         s2 = "O {}".format(ort)
-        return ProgressBarFancy.get_d(s1, s2, width, lp)  
+        return ProgressBarFancy.get_d(s1, s2, width, lp, lps)  
 
     @staticmethod
-    def reduced_2_stat(p, tet, speed, ttg, eta, ort, repl_ch, width, lp):
+    def reduced_2_stat(p, tet, speed, ttg, eta, ort, repl_ch, width, lp, lps):
         s1 = "E {} G {}".format(tet, ttg)
         s2 = "O {}".format(ort)
-        return ProgressBarFancy.get_d(s1, s2, width, lp)
+        return ProgressBarFancy.get_d(s1, s2, width, lp, lps)
     
     @staticmethod
-    def reduced_3_stat(p, tet, speed, ttg, eta, ort, repl_ch, width, lp):
+    def reduced_3_stat(p, tet, speed, ttg, eta, ort, repl_ch, width, lp, lps):
         s1 = "E {} G {}".format(tet, ttg)
         s2 = ''
-        return ProgressBarFancy.get_d(s1, s2, width, lp)
+        return ProgressBarFancy.get_d(s1, s2, width, lp, lps)
+    
+    @staticmethod
+    def reduced_4_stat(p, tet, speed, ttg, eta, ort, repl_ch, width, lp, lps):
+        s1 = ''
+        s2 = ''
+        return ProgressBarFancy.get_d(s1, s2, width, lp, lps)    
 
     @staticmethod        
     def kw_bold(s, ch_after):
@@ -955,14 +961,20 @@ class ProgressBarFancy(Progress):
             # only show current absolute progress as number and estimated speed
             print("{}{} [{}] #{}    ".format(prepend, humanize_time(tet), humanize_speed(speed), count_value))             
         else:
+            if width == 'auto':
+                width = get_terminal_width()
             # deduce relative progress
             p = count_value / max_count_value
+            if p < 1:
+                ps = " {:.1%} ".format(p)
+            else:
+                ps = " {:.0%} ".format(p)
             
             if ttg is None:
                 eta = '--'
                 ort = None
             else:
-                eta = datetime.datetime.fromtimestamp(time.time() + ttg).strftime("%Y-%m-%d_%H:%M:%S")
+                eta = datetime.datetime.fromtimestamp(time.time() + ttg).strftime("%Y%m%d_%H:%M:%S")
                 ort = tet + ttg
                 
             tet = humanize_time(tet)
@@ -972,29 +984,37 @@ class ProgressBarFancy(Progress):
             repl_ch = '-'
             lp = len(prepend)
             
-            res = ProgressBarFancy.full_stat(p, tet, speed, ttg, eta, ort, repl_ch, width, lp)
+            args = p, tet, speed, ttg, eta, ort, repl_ch, width, lp, len(ps)
+            
+            res = ProgressBarFancy.full_stat(*args)
             if res is None:
-                res = ProgressBarFancy.full_minor_stat(p, tet, speed, ttg, eta, ort, repl_ch, width, lp)
+                res = ProgressBarFancy.full_minor_stat(*args)
                 if res is None:
-                    res = ProgressBarFancy.reduced_1_stat(p, tet, speed, ttg, eta, ort, repl_ch, width, lp)
+                    res = ProgressBarFancy.reduced_1_stat(*args)
                     if res is None:
-                        res = ProgressBarFancy.reduced_2_stat(p, tet, speed, ttg, eta, ort, repl_ch, width, lp)
+                        res = ProgressBarFancy.reduced_2_stat(*args)
                         if res is None:
-                            res = ProgressBarFancy.reduced_3_stat(p, tet, speed, ttg, eta, ort, repl_ch, width, lp)
+                            res = ProgressBarFancy.reduced_3_stat(*args)
                             if res is None:
-                                res = '', '', width-2-len(prepend)
+                                res = ProgressBarFancy.reduced_4_stat(*args)
                                     
-            s1, s2, d = res                
-            s = "{0}{2}{1}".format(s1, s2, ' '*d)
-            s_before = s[:math.ceil(width*p)].replace(' ', repl_ch)
-            if (len(s_before) > 0) and (s_before[-1] == repl_ch):
-                s_before = s_before[:-1] + '>'
-            s_after  = s[math.ceil(width*p):]
+            if res is not None:
+                s1, s2, d1, d2 = res                
+                s = s1 + ' '*d1 + ps + ' '*d2 + s2
                    
-            s_before = ProgressBarFancy.kw_bold(s_before, ch_after=[repl_ch, '>'])
-            s_after = ProgressBarFancy.kw_bold(s_after, ch_after=[' '])
-            print(prepend + ESC_BOLD + '[' + ESC_RESET_BOLD + ESC_LIGHT_GREEN + s_before + ESC_DEFAULT + s_after + ESC_BOLD + ']' + ESC_NO_CHAR_ATTR)
+                s_before = s[:math.ceil(width*p)].replace(' ', repl_ch)
+                if (len(s_before) > 0) and (s_before[-1] == repl_ch):
+                    s_before = s_before[:-1] + '>'
+                s_after  = s[math.ceil(width*p):]
                 
+                s_before = ProgressBarFancy.kw_bold(s_before, ch_after=[repl_ch, '>'])
+                s_after = ProgressBarFancy.kw_bold(s_after, ch_after=[' '])
+                print(prepend + ESC_BOLD + '[' + ESC_RESET_BOLD + ESC_LIGHT_GREEN + s_before + ESC_DEFAULT + s_after + ESC_BOLD + ']' + ESC_NO_CHAR_ATTR)
+            else:
+                ps = ps.strip()
+                if p == 1:
+                    ps = ' '+ps
+                print(prepend + ps)
             
             
                 
@@ -1301,13 +1321,12 @@ def remove_ESC_SEQ_from_string(s):
     return s
 
 
-def terminal_reserve():
+def terminal_reserve(progress_obj, terminal_obj=None, verbose=0, identifier=None):
     """ Registers the terminal (stdout) for printing.
     
     Useful to prevent multiple processes from writing progress bars
     to stdout.
     
-    It is currently handled with a simple list.
     One process (server) prints to stdout and a couple of subprocesses
     do not print to the same stdout, because the server has reserved it.
     Of course, the clients have to be nice and check with 
@@ -1316,22 +1335,41 @@ def terminal_reserve():
     
     Returns
     -------
-    True if reservation was successfull, false if there already is a
-    reservation.
+    True if reservation was successful (or if we have already reserved this tty),
+    False if there already is a reservation from another instance.
     """
-    for term in TERMINAL_RESERVATION:
-        if sys.stdout is term:
-            # someone else is using this stdout
+    if terminal_obj is None:
+        terminal_obj = sys.stdout
+    
+    if identifier is None:
+        identifier = ''
+    else:
+        identifier = identifier + ': ' 
+    
+    if terminal_obj in TERMINAL_RESERVATION:    # terminal was already registered
+        if verbose > 1:
+            print("{}this terminal {} has already been added to reservation list".format(identifier, terminal_obj))
+        
+        if TERMINAL_RESERVATION[terminal_obj] is progress_obj:
+            if verbose > 1:
+                print("{}we {} have already reserved this terminal {}".format(identifier, progress_obj, terminal_obj))
+            return True
+        else:
+            if verbose > 1:
+                print("{}someone else {} has already reserved this terminal {}".format(identifier, TERMINAL_RESERVATION[terminal_obj], terminal_obj))
             return False
-    
-    # we have now registered this stdout
-    TERMINAL_RESERVATION.append(sys.stdout)
-    return True
+    else:                                       # terminal not yet registered
+        if verbose > 1:
+            print("{}terminal {} was reserved for us {}".format(identifier, terminal_obj, progress_obj))
+        TERMINAL_RESERVATION[terminal_obj] = progress_obj
+        return True
 
 
-def terminal_unreserve():
+def terminal_unreserve(progress_obj, terminal_obj=None, verbose=0, identifier=None):
     """ Unregisters the terminal (stdout) for printing.
     
+    an instance (progress_obj) can only unreserve the tty (terminal_obj) when it also reserved it
+    
     see terminal_reserved for more information
     
     Returns
@@ -1339,12 +1377,26 @@ def terminal_unreserve():
     None
     """
     
-    for term in TERMINAL_RESERVATION:
-        if sys.stdout is term:
-            TERMINAL_RESERVATION.remove(term)
-            return None
+    if terminal_obj is None:
+        terminal_obj =sys.stdout
 
+    if identifier is None:
+        identifier = ''
+    else:
+        identifier = identifier + ': '         
     
+    po = TERMINAL_RESERVATION.get(terminal_obj)
+    if po is None:
+        if verbose > 1:
+            print("{}terminal {} was not reserved, nothing happens".format(identifier, terminal_obj))
+    else:
+        if po is progress_obj:
+            if verbose > 1:
+                print("{}terminal {} now unreserned".format(identifier, terminal_obj))
+            del TERMINAL_RESERVATION[terminal_obj]
+        else:
+            if verbose > 1:
+                print("{}you {} can NOT unreserve terminal {} be cause it was reserved by {}".format(identifier, progress_obj, terminal_obj, po))
 
 
 myQueue = mp.Queue
@@ -1426,6 +1478,6 @@ ESC_SEQ_SET = [ESC_NO_CHAR_ATTR,
                ESC_WHITE]
 
 # terminal reservation list, see terminal_reserve
-TERMINAL_RESERVATION = []
+TERMINAL_RESERVATION = {}
 # these are classes that print progress bars, see terminal_reserve
 TERMINAL_PRINT_LOOP_CLASSES = ["ProgressBar", "ProgressBarCounter"]

tests/test_progress.py

@@ -283,6 +283,7 @@ def test_progress_bar():
     time.sleep(2)
     
 def test_progress_bar_with_statement():
+    print("TERMINAL_RESERVATION", progress.TERMINAL_RESERVATION)
     count = progress.UnsignedIntValue()
     max_count = progress.UnsignedIntValue(100)
     with progress.ProgressBar(count, max_count, verbose=2) as sb:
@@ -304,6 +305,7 @@ def test_progress_bar_with_statement():
     sb.stop()
     
 def test_progress_bar_multi():
+    print("TERMINAL_RESERVATION", progress.TERMINAL_RESERVATION)
     n = 4
     max_count_value = 100
     
@@ -391,7 +393,9 @@ def test_intermediate_prints_while_running_progess_bar():
                     c.value += 1
                     
                 if c.value == 100:
+                    sc.stop()
                     print("intermediate message")
+                    sc.start()
                     
                 if c.value == 400:
                     break
@@ -415,9 +419,9 @@ def test_intermediate_prints_while_running_progess_bar_multi():
                 c[i].value += 1
                 
             if c[0].value == 100:
+                sc.stop()
                 print("intermediate message")
-                with c[i].get_lock():
-                    c[i].value += 1
+                sc.start()
                 
             if c[0].value == 400:
                 break
@@ -550,11 +554,11 @@ def test_progress_bar_start_stop():
 def test_progress_bar_fancy():
     count = progress.UnsignedIntValue()
     max_count = progress.UnsignedIntValue(100)
-    with progress.ProgressBarFancy(count, max_count, verbose=1, interval=0.2, width=80) as sb:
+    with progress.ProgressBarFancy(count, max_count, verbose=1, interval=0.2, width='auto') as sb:
         sb.start()
         for i in range(100):
             count.value = i+1
-            time.sleep(0.1)
+            time.sleep(0.3)
  
 def test_progress_bar_multi_fancy():
     n = 4
@@ -593,7 +597,7 @@ def test_progress_bar_multi_fancy():
             
 def test_progress_bar_fancy_small():
     count = progress.UnsignedIntValue()
-    m = 30
+    m = 15
     max_count = progress.UnsignedIntValue(m)
     with progress.ProgressBarFancy(count, max_count, verbose=1, interval=0.2, width='auto') as sb:
         sb.start()
@@ -643,29 +647,41 @@ def test_progress_bar_fancy_small():
             count.value = i+1
             time.sleep(0.1)
  
+    with progress.ProgressBarFancy(count, max_count, verbose=1, interval=0.2, width=10) as sb:
+        sb.start()
+        for i in range(m):
+            count.value = i+1
+            time.sleep(0.1)
+            
+    with progress.ProgressBarFancy(count, max_count, verbose=1, interval=0.2, width=5) as sb:
+        sb.start()
+        for i in range(m):
+            count.value = i+1
+            time.sleep(0.1)
+            
 if __name__ == "__main__":
     func = [    
-#     test_loop_basic,
-#     test_loop_signals,
-#     test_loop_normal_stop,
-#     test_loop_need_sigterm_to_stop,
-#     test_loop_need_sigkill_to_stop,
-#     test_why_with_statement,
-#     test_progress_bar,
-#     test_progress_bar_with_statement,
-#     test_progress_bar_multi,
-#     test_status_counter,
-#     test_status_counter_multi,
-#     test_intermediate_prints_while_running_progess_bar,
-#     test_intermediate_prints_while_running_progess_bar_multi,
-#     test_progress_bar_counter,
-#     test_progress_bar_counter_non_max,
-#     test_progress_bar_counter_hide_bar,
-#     test_progress_bar_slow_change,
-#     test_progress_bar_start_stop,
+    test_loop_basic,
+    test_loop_signals,
+    test_loop_normal_stop,
+    test_loop_need_sigterm_to_stop,
+    test_loop_need_sigkill_to_stop,
+    test_why_with_statement,
+    test_progress_bar,
+    test_progress_bar_with_statement,
+    test_progress_bar_multi,
+    test_status_counter,
+    test_status_counter_multi,
+    test_intermediate_prints_while_running_progess_bar,
+    test_intermediate_prints_while_running_progess_bar_multi,
+    test_progress_bar_counter,
+    test_progress_bar_counter_non_max,
+    test_progress_bar_counter_hide_bar,
+    test_progress_bar_slow_change,
+    test_progress_bar_start_stop,
     test_progress_bar_fancy,
-#     test_progress_bar_multi_fancy,
-#     test_progress_bar_fancy_small,
+    test_progress_bar_multi_fancy,
+    test_progress_bar_fancy_small,
     lambda: print("END")
     ]