hiro/jobmanager
1
0
Fork 0
mirror of https://github.com/vale981/jobmanager synced 2025-03-06 02:11:39 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

put communication to server in client class only, was in workerfunction before. so the client machiene has to fetch possibly huge data only once

Browse source
This commit is contained in:
Richard Hartmann 2014-09-26 16:51:20 +02:00
parent 7291233a5a
commit 3bbeecd75e
2 changed files with 49 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
jobmanager.py
View file

@ -1109,9 +1109,18 @@ class JobManager_Client(object):
self.procs = []
self.manager_objects = self.get_manager_objects()
def get_manager_objects(self):
return JobManager_Client._get_manager_objects(self.server,
self.port,
self.authkey,
self._identifier,
self.verbose)
@staticmethod
def _get_manager_object(server, port, authkey, identifier, verbose=0):
def _get_manager_objects(server, port, authkey, identifier, verbose=0):
"""
connects to the server and get registered shared objects such as
job_q, result_q, fail_q, const_arg
@ -1174,20 +1183,21 @@ class JobManager_Client(object):
@staticmethod
def __worker_func(func, nice, verbose, server, port, authkey, i):
def __worker_func(func, nice, verbose, server, port, authkey, i, manager_objects=None):
"""
the wrapper spawned nproc trimes calling and handling self.func
"""
identifier = get_identifier(name='worker{}'.format(i+1))
Signal_to_sys_exit(signals=[signal.SIGTERM, signal.SIGINT])
res = JobManager_Client._get_manager_object(server, port, authkey, identifier, verbose)
if res == None:
if verbose > 1:
print("{}: no shared object recieved, terminate!".format(identifier))
sys.exit(1)
else:
job_q, result_q, fail_q, const_arg = res
if manager_objects is None:
manager_objects = JobManager_Client._get_manager_object(server, port, authkey, identifier, verbose)
if res == None:
if verbose > 1:
print("{}: no shared object recieved, terminate!".format(identifier))
sys.exit(1)
job_q, result_q, fail_q, const_arg = manager_objects
n = os.nice(0)
n = os.nice(nice - n)
@ -1266,8 +1276,10 @@ class JobManager_Client(object):
print(" continue processing next argument.")
if verbose > 0:
print("{}: calculation:{:.2%} communication:{:.2%}".format(identifier, time_calc/(time_calc+time_queue), time_queue/(time_calc+time_queue)))
try:
print("{}: calculation:{:.2%} communication:{:.2%}".format(identifier, time_calc/(time_calc+time_queue), time_queue/(time_calc+time_queue)))
except:
pass
if verbose > 1:
print("{}: JobManager_Client.__worker_func terminates".format(identifier))
@ -1289,7 +1301,8 @@ class JobManager_Client(object):
self.server,
self.port,
self.authkey,
i))
i,
self.manager_objects))
self.procs.append(p)
p.start()
time.sleep(0.3)

46
test_jobmanager.py
View file

@ -518,7 +518,7 @@ def shutdown_client(sig):
check if the final_result contain all arguments given
"""
n = 100
n = 300
print("## terminate client with {} ##".format(jobmanager.signal_dict[sig]))
@ -530,7 +530,7 @@ def shutdown_client(sig):
p_client = mp.Process(target=start_client)
p_client.start()
time.sleep(1)
time.sleep(5)
print(" send {}".format(jobmanager.signal_dict[sig]))
os.kill(p_client.pid, sig)
@ -746,28 +746,28 @@ def test_hashedViewOnNumpyArray():
if __name__ == "__main__":
test_Signal_to_SIG_IGN()
test_Signal_to_sys_exit()
test_Signal_to_terminate_process_list()
test_loop_basic()
test_loop_signals()
test_loop_normal_stop()
test_loop_need_sigterm_to_stop()
test_loop_need_sigkill_to_stop()
test_why_with_statement()
# test_Signal_to_SIG_IGN()
# test_Signal_to_sys_exit()
# test_Signal_to_terminate_process_list()
#
# test_loop_basic()
# test_loop_signals()
# test_loop_normal_stop()
# test_loop_need_sigterm_to_stop()
# test_loop_need_sigkill_to_stop()
#
# test_why_with_statement()
#
# test_statusbar()
# test_statusbar_with_statement()
test_statusbar()
test_statusbar_with_statement()
test_jobmanager_basic()
test_jobmanager_server_signals()
test_shutdown_server_while_client_running()
# test_jobmanager_basic()
# test_jobmanager_server_signals()
# test_shutdown_server_while_client_running()
test_shutdown_client()
test_check_fail()
test_jobmanager_read_old_stat()
test_hashDict()
test_hashedViewOnNumpyArray()
# test_check_fail()
# test_jobmanager_read_old_stat()
# test_hashDict()
# test_hashedViewOnNumpyArray()
pass