mirror of
https://github.com/vale981/jobmanager
synced 2025-03-06 10:21:39 -05:00
1682 lines
No EOL
103 KiB
HTML
1682 lines
No EOL
103 KiB
HTML
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
|
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
|
|
|
|
|
<html xmlns="http://www.w3.org/1999/xhtml">
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
|
|
<title>jobmanager documentation — jobmanager 0.1.0 documentation</title>
|
|
|
|
<link rel="stylesheet" href="_static/default.css" type="text/css" />
|
|
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
|
|
|
|
<script type="text/javascript">
|
|
var DOCUMENTATION_OPTIONS = {
|
|
URL_ROOT: './',
|
|
VERSION: '0.1.0',
|
|
COLLAPSE_INDEX: false,
|
|
FILE_SUFFIX: '.html',
|
|
HAS_SOURCE: true
|
|
};
|
|
</script>
|
|
<script type="text/javascript" src="_static/jquery.js"></script>
|
|
<script type="text/javascript" src="_static/underscore.js"></script>
|
|
<script type="text/javascript" src="_static/doctools.js"></script>
|
|
<script type="text/javascript" src="http://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
|
|
<link rel="top" title="jobmanager 0.1.0 documentation" href="#" />
|
|
<link rel="next" title="jobmanager documentation" href="" />
|
|
</head>
|
|
<body>
|
|
<div class="related">
|
|
<h3>Navigation</h3>
|
|
<ul>
|
|
<li class="right" style="margin-right: 10px">
|
|
<a href="genindex.html" title="General Index"
|
|
accesskey="I">index</a></li>
|
|
<li class="right" >
|
|
<a href="py-modindex.html" title="Python Module Index"
|
|
>modules</a> |</li>
|
|
<li class="right" >
|
|
<a href="#" title="jobmanager documentation"
|
|
accesskey="N">next</a> |</li>
|
|
<li><a href="#">jobmanager 0.1.0 documentation</a> »</li>
|
|
</ul>
|
|
</div>
|
|
|
|
<div class="document">
|
|
<div class="documentwrapper">
|
|
<div class="bodywrapper">
|
|
<div class="body">
|
|
|
|
<div class="section" id="jobmanager-documentation">
|
|
<h1>jobmanager documentation<a class="headerlink" href="#jobmanager-documentation" title="Permalink to this headline">¶</a></h1>
|
|
<div class="section" id="content">
|
|
<h2>Content<a class="headerlink" href="#content" title="Permalink to this headline">¶</a></h2>
|
|
<div class="toctree-wrapper compound">
|
|
<ul class="current">
|
|
<li class="toctree-l1 current"><a class="current reference internal" href="">jobmanager documentation</a><ul>
|
|
<li class="toctree-l2"><a class="reference internal" href="#content">Content</a><ul class="simple">
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="#module-jobmanager">Main module</a></li>
|
|
<li class="toctree-l2"><a class="reference internal" href="#module-jobmanager.progress">Progress monitoring</a><ul class="simple">
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="#user-convenience">User Convenience</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="#module-jobmanager.decorators">Decorators</a><ul class="simple">
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l3"><a class="reference internal" href="#module-jobmanager.ode_wrapper">ODE wrapper</a></li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="#core-modules">Core modules</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="#jobmanager">jobmanager</a></li>
|
|
<li class="toctree-l3"><a class="reference internal" href="#module-jobmanager.jobmanager">clients</a><ul class="simple">
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l3"><a class="reference internal" href="#id7">clients</a><ul class="simple">
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l3"><a class="reference internal" href="#module-jobmanager.servers">servers</a></li>
|
|
<li class="toctree-l3"><a class="reference internal" href="#module-jobmanager.persistentData">persistentData</a><ul class="simple">
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
<li class="toctree-l2"><a class="reference internal" href="#examples">Examples</a><ul>
|
|
<li class="toctree-l3"><a class="reference internal" href="#module-simple_example">simple example</a></li>
|
|
<li class="toctree-l3"><a class="reference internal" href="#id10">decorators</a></li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
<p>For an alphabetically sorted list of all modules, classes, and
|
|
methods, see <a class="reference internal" href="genindex.html"><em>Index</em></a>.</p>
|
|
</div>
|
|
<div class="section" id="module-jobmanager">
|
|
<span id="main-module"></span><h2>Main module<a class="headerlink" href="#module-jobmanager" title="Permalink to this headline">¶</a></h2>
|
|
<p>Easy distributed computing based on the python class SyncManager for remote communication and python module multiprocessing for local parallelism.</p>
|
|
<p>Scheduling across different processes/machines is implemented in the
|
|
core modules <a class="reference internal" href="#module-jobmanager.jobmanager" title="jobmanager.jobmanager"><tt class="xref py py-mod docutils literal"><span class="pre">jobmanager.jobmanager</span></tt></a>, <a class="reference internal" href="#module-jobmanager.servers" title="jobmanager.servers"><tt class="xref py py-mod docutils literal"><span class="pre">jobmanager.servers</span></tt></a>,
|
|
and <a class="reference internal" href="#module-jobmanager.clients" title="jobmanager.clients"><tt class="xref py py-mod docutils literal"><span class="pre">jobmanager.clients</span></tt></a>.</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><a class="reference internal" href="#jobmanager.jobmanager.JobManager_Client" title="jobmanager.jobmanager.JobManager_Client"><tt class="xref py py-obj docutils literal"><span class="pre">JobManager_Client</span></tt></a>(server, authkey[, port, ...])</td>
|
|
<td>Calls the functions self.func with arguments fetched from the job_q.</td>
|
|
</tr>
|
|
<tr class="row-even"><td><a class="reference internal" href="#jobmanager.jobmanager.JobManager_Server" title="jobmanager.jobmanager.JobManager_Server"><tt class="xref py py-obj docutils literal"><span class="pre">JobManager_Server</span></tt></a>(authkey[, const_arg, ...])</td>
|
|
<td>general usage:</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<div class="figure align-center">
|
|
<img alt="_images/server_client_communication.png" src="_images/server_client_communication.png" />
|
|
</div>
|
|
<p>Progress classes are implemented in the <a class="reference internal" href="#module-jobmanager.progress" title="jobmanager.progress"><tt class="xref py py-mod docutils literal"><span class="pre">jobmanager.progress</span></tt></a>
|
|
submodule. Intuitive access to progress bars is facilitated with
|
|
decorators (<a class="reference internal" href="#module-jobmanager.decorators" title="jobmanager.decorators"><tt class="xref py py-mod docutils literal"><span class="pre">jobmanager.decorators</span></tt></a>).</p>
|
|
</div>
|
|
<div class="section" id="module-jobmanager.progress">
|
|
<span id="progress-monitoring"></span><h2>Progress monitoring<a class="headerlink" href="#module-jobmanager.progress" title="Permalink to this headline">¶</a></h2>
|
|
<dl class="class">
|
|
<dt id="jobmanager.progress.Loop">
|
|
<em class="property">class </em><tt class="descclassname">jobmanager.progress.</tt><tt class="descname">Loop</tt><big>(</big><em>func</em>, <em>args=()</em>, <em>interval=1</em>, <em>verbose=0</em>, <em>sigint='stop'</em>, <em>sigterm='stop'</em>, <em>name=None</em>, <em>auto_kill_on_last_resort=False</em><big>)</big><a class="reference internal" href="_modules/jobmanager/progress.html#Loop"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.Loop" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>class to run a function periodically an seperate process.</p>
|
|
<p>In case the called function returns True, the loop will stop.
|
|
Otherwise a time interval given by interval will be slept before
|
|
another execution is triggered.</p>
|
|
<p>The shared memory variable _run (accessible via the class property run)
|
|
also determines if the function if executed another time. If set to False
|
|
the execution stops.</p>
|
|
<p>For safe cleanup (and in order to catch any Errors)
|
|
it is advisable to instantiate this class
|
|
using ‘with’ statement as follows:</p>
|
|
<blockquote>
|
|
<div><dl class="docutils">
|
|
<dt>with Loop(<a href="#id1"><span class="problematic" id="id2">**</span></a>kwargs) as my_loop:</dt>
|
|
<dd>my_loop.start()
|
|
...</dd>
|
|
</dl>
|
|
</div></blockquote>
|
|
<p>this will guarantee you that the spawned loop process is
|
|
down when exiting the ‘with’ scope.</p>
|
|
<p>The only circumstance where the process is still running is
|
|
when you set auto_kill_on_last_resort to False and answer the
|
|
question to send SIGKILL with no.</p>
|
|
<p class="rubric">Attributes</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><a class="reference internal" href="#jobmanager.progress.Loop.run" title="jobmanager.progress.Loop.run"><tt class="xref py py-obj docutils literal"><span class="pre">run</span></tt></a></td>
|
|
<td>makes the shared memory boolean accessible as class attribute</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<p class="rubric">Methods</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><a class="reference internal" href="#jobmanager.progress.Loop.getpid" title="jobmanager.progress.Loop.getpid"><tt class="xref py py-obj docutils literal"><span class="pre">getpid</span></tt></a>()</td>
|
|
<td>return the process id of the spawned process</td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">is_alive</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><a class="reference internal" href="#jobmanager.progress.Loop.join" title="jobmanager.progress.Loop.join"><tt class="xref py py-obj docutils literal"><span class="pre">join</span></tt></a>(timeout)</td>
|
|
<td>calls join for the spawned process with given timeout</td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">pause</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">resume</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><a class="reference internal" href="#jobmanager.progress.Loop.start" title="jobmanager.progress.Loop.start"><tt class="xref py py-obj docutils literal"><span class="pre">start</span></tt></a>()</td>
|
|
<td>uses multiprocess Process to call _wrapper_func in subprocess</td>
|
|
</tr>
|
|
<tr class="row-odd"><td><a class="reference internal" href="#jobmanager.progress.Loop.stop" title="jobmanager.progress.Loop.stop"><tt class="xref py py-obj docutils literal"><span class="pre">stop</span></tt></a>()</td>
|
|
<td>stops the process triggered by start</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<dl class="method">
|
|
<dt id="jobmanager.progress.Loop.getpid">
|
|
<tt class="descname">getpid</tt><big>(</big><big>)</big><a class="reference internal" href="_modules/jobmanager/progress.html#Loop.getpid"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.Loop.getpid" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>return the process id of the spawned process</p>
|
|
</dd></dl>
|
|
|
|
<dl class="method">
|
|
<dt id="jobmanager.progress.Loop.join">
|
|
<tt class="descname">join</tt><big>(</big><em>timeout</em><big>)</big><a class="reference internal" href="_modules/jobmanager/progress.html#Loop.join"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.Loop.join" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>calls join for the spawned process with given timeout</p>
|
|
</dd></dl>
|
|
|
|
<dl class="attribute">
|
|
<dt id="jobmanager.progress.Loop.run">
|
|
<tt class="descname">run</tt><a class="reference internal" href="_modules/jobmanager/progress.html#Loop.run"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.Loop.run" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>makes the shared memory boolean accessible as class attribute</p>
|
|
<p>Set run False, the loop will stop repeating.
|
|
Calling start, will set run True and start the loop again as a new process.</p>
|
|
</dd></dl>
|
|
|
|
<dl class="method">
|
|
<dt id="jobmanager.progress.Loop.start">
|
|
<tt class="descname">start</tt><big>(</big><big>)</big><a class="reference internal" href="_modules/jobmanager/progress.html#Loop.start"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.Loop.start" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>uses multiprocess Process to call _wrapper_func in subprocess</p>
|
|
</dd></dl>
|
|
|
|
<dl class="method">
|
|
<dt id="jobmanager.progress.Loop.stop">
|
|
<tt class="descname">stop</tt><big>(</big><big>)</big><a class="reference internal" href="_modules/jobmanager/progress.html#Loop.stop"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.Loop.stop" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>stops the process triggered by start</p>
|
|
<p>Setting the shared memory boolean run to false, which should prevent
|
|
the loop from repeating. Call __cleanup to make sure the process
|
|
stopped. After that we could trigger start() again.</p>
|
|
</dd></dl>
|
|
|
|
</dd></dl>
|
|
|
|
<dl class="class">
|
|
<dt id="jobmanager.progress.Progress">
|
|
<em class="property">class </em><tt class="descclassname">jobmanager.progress.</tt><tt class="descname">Progress</tt><big>(</big><em>count</em>, <em>max_count=None</em>, <em>prepend=None</em>, <em>width='auto'</em>, <em>speed_calc_cycles=10</em>, <em>interval=1</em>, <em>verbose=0</em>, <em>sigint='stop'</em>, <em>sigterm='stop'</em>, <em>name='progress'</em><big>)</big><a class="reference internal" href="_modules/jobmanager/progress.html#Progress"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.Progress" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>Abstract Progress Loop</p>
|
|
<p>Uses Loop class to implement a repeating function to display the progress
|
|
of multiples processes.</p>
|
|
<p>In the simplest case, given only a list of shared memory values ‘count’ (NOTE:
|
|
a single value will be automatically mapped to a one element list),
|
|
the total elapses time (TET) and a speed estimation are calculated for
|
|
each process and passed to the display function show_stat.</p>
|
|
<p>This functions needs to be implemented in a subclass. It is intended to show
|
|
the progress of ONE process in one line.</p>
|
|
<p>When max_count is given (in general as list of shared memory values, a single
|
|
shared memory value will be mapped to a one element list) the time to go TTG
|
|
will also be calculated and passed tow show_stat.</p>
|
|
<p>Information about the terminal width will be retrieved when setting width=’auto’.
|
|
If supported by the terminal emulator the width in characters of the terminal
|
|
emulator will be stored in width and also passed to show_stat.
|
|
Otherwise, a default width of 80 characters will be chosen.</p>
|
|
<p>Also you can specify a fixed width by setting width to the desired number.</p>
|
|
<p>NOTE: in order to achieve multiline progress special escape sequences are used
|
|
which may not be supported by all terminal emulators.</p>
|
|
<p>example:</p>
|
|
<blockquote>
|
|
<div><p>c1 = UnsignedIntValue(val=0)
|
|
c2 = UnsignedIntValue(val=0)</p>
|
|
<p>c = [c1, c2]
|
|
prepend = [‘c1: ‘, ‘c2: ‘]
|
|
with ProgressCounter(count=c, prepend=prepend) as sc:</p>
|
|
<blockquote>
|
|
<div><p>sc.start()
|
|
while True:</p>
|
|
<blockquote>
|
|
<div><p>i = np.random.randint(0,2)
|
|
with c[i].get_lock():</p>
|
|
<blockquote>
|
|
<div>c[i].value += 1</div></blockquote>
|
|
<dl class="docutils">
|
|
<dt>if c[0].value == 1000:</dt>
|
|
<dd>break</dd>
|
|
</dl>
|
|
<p>time.sleep(0.01)</p>
|
|
</div></blockquote>
|
|
</div></blockquote>
|
|
</div></blockquote>
|
|
<p>using start() within the ‘with’ statement ensures that the subprocess
|
|
which is started by start() in order to show the progress repeatedly
|
|
will be terminated on context exit. Otherwise one has to make sure
|
|
that at some point the stop() routine is called. When dealing with
|
|
signal handling and exception this might become tricky, so that the
|
|
use of the ‘with’ statement is highly encouraged.</p>
|
|
<p class="rubric">Attributes</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">run</span></tt></td>
|
|
<td>makes the shared memory boolean accessible as class attribute</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<p class="rubric">Methods</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">getpid</span></tt>()</td>
|
|
<td>return the process id of the spawned process</td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">is_alive</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">join</span></tt>(timeout)</td>
|
|
<td>calls join for the spawned process with given timeout</td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">pause</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><a class="reference internal" href="#jobmanager.progress.Progress.reset" title="jobmanager.progress.Progress.reset"><tt class="xref py py-obj docutils literal"><span class="pre">reset</span></tt></a>([i])</td>
|
|
<td>convenient function to reset progress information</td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">resume</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><a class="reference internal" href="#jobmanager.progress.Progress.show_stat" title="jobmanager.progress.Progress.show_stat"><tt class="xref py py-obj docutils literal"><span class="pre">show_stat</span></tt></a>(count_value, max_count_value, ...)</td>
|
|
<td>re implement this function in a subclass</td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">show_stat_wrapper</span></tt>(count, last_count, ...)</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><a class="reference internal" href="#jobmanager.progress.Progress.show_stat_wrapper_multi" title="jobmanager.progress.Progress.show_stat_wrapper_multi"><tt class="xref py py-obj docutils literal"><span class="pre">show_stat_wrapper_multi</span></tt></a>(count, last_count, ...)</td>
|
|
<td>call the static method show_stat_wrapper for each process</td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">start</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><a class="reference internal" href="#jobmanager.progress.Progress.stop" title="jobmanager.progress.Progress.stop"><tt class="xref py py-obj docutils literal"><span class="pre">stop</span></tt></a>([make_sure_its_down])</td>
|
|
<td>trigger clean up by hand, needs to be done when not using</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<dl class="method">
|
|
<dt id="jobmanager.progress.Progress.reset">
|
|
<tt class="descname">reset</tt><big>(</big><em>i=None</em><big>)</big><a class="reference internal" href="_modules/jobmanager/progress.html#Progress.reset"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.Progress.reset" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>convenient function to reset progress information</p>
|
|
<p>i [None, int] - None: reset all, int: reset process indexed by i</p>
|
|
</dd></dl>
|
|
|
|
<dl class="staticmethod">
|
|
<dt id="jobmanager.progress.Progress.show_stat">
|
|
<em class="property">static </em><tt class="descname">show_stat</tt><big>(</big><em>count_value</em>, <em>max_count_value</em>, <em>prepend</em>, <em>speed</em>, <em>tet</em>, <em>ttg</em>, <em>width</em>, <em>**kwargs</em><big>)</big><a class="reference internal" href="_modules/jobmanager/progress.html#Progress.show_stat"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.Progress.show_stat" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>re implement this function in a subclass</p>
|
|
<p>count_value - current value of progress</p>
|
|
<p>max_count_value - maximum value the progress can take</p>
|
|
<p>prepend - some extra string to be put for example in front of the
|
|
progress display</p>
|
|
<p>speed - estimated speed in counts per second (use for example humanize_speed
|
|
to get readable information in string format)</p>
|
|
<p>tet - total elapsed time in seconds (use for example humanize_time
|
|
to get readable information in string format)</p>
|
|
<p>ttg - time to go in seconds (use for example humanize_time
|
|
to get readable information in string format)</p>
|
|
</dd></dl>
|
|
|
|
<dl class="staticmethod">
|
|
<dt id="jobmanager.progress.Progress.show_stat_wrapper_multi">
|
|
<em class="property">static </em><tt class="descname">show_stat_wrapper_multi</tt><big>(</big><em>count</em>, <em>last_count</em>, <em>start_time</em>, <em>max_count</em>, <em>speed_calc_cycles</em>, <em>width</em>, <em>q</em>, <em>last_old_count</em>, <em>last_old_time</em>, <em>prepend</em>, <em>show_stat_function</em>, <em>len_</em>, <em>add_args</em>, <em>lock</em><big>)</big><a class="reference internal" href="_modules/jobmanager/progress.html#Progress.show_stat_wrapper_multi"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.Progress.show_stat_wrapper_multi" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>call the static method show_stat_wrapper for each process</p>
|
|
</dd></dl>
|
|
|
|
<dl class="method">
|
|
<dt id="jobmanager.progress.Progress.stop">
|
|
<tt class="descname">stop</tt><big>(</big><em>make_sure_its_down=False</em><big>)</big><a class="reference internal" href="_modules/jobmanager/progress.html#Progress.stop"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.Progress.stop" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>trigger clean up by hand, needs to be done when not using
|
|
context management via ‘with’ statement</p>
|
|
<ul class="simple">
|
|
<li>will terminate loop process</li>
|
|
<li>show a last progress -> see the full 100% on exit</li>
|
|
<li>releases terminal reservation</li>
|
|
</ul>
|
|
</dd></dl>
|
|
|
|
</dd></dl>
|
|
|
|
<dl class="class">
|
|
<dt id="jobmanager.progress.ProgressBar">
|
|
<em class="property">class </em><tt class="descclassname">jobmanager.progress.</tt><tt class="descname">ProgressBar</tt><big>(</big><em>count</em>, <em>max_count=None</em>, <em>width='auto'</em>, <em>prepend=None</em>, <em>speed_calc_cycles=10</em>, <em>interval=1</em>, <em>verbose=0</em>, <em>sigint='stop'</em>, <em>sigterm='stop'</em>, <em>name='progress_bar'</em><big>)</big><a class="reference internal" href="_modules/jobmanager/progress.html#ProgressBar"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.ProgressBar" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>implements a progress bar similar to the one known from ‘wget’ or ‘pv’</p>
|
|
<p class="rubric">Attributes</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">run</span></tt></td>
|
|
<td>makes the shared memory boolean accessible as class attribute</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<p class="rubric">Methods</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">getpid</span></tt>()</td>
|
|
<td>return the process id of the spawned process</td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">is_alive</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">join</span></tt>(timeout)</td>
|
|
<td>calls join for the spawned process with given timeout</td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">pause</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">reset</span></tt>([i])</td>
|
|
<td>convenient function to reset progress information</td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">resume</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">show_stat</span></tt>(count_value, max_count_value, ...)</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">show_stat_wrapper</span></tt>(count, last_count, ...)</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">show_stat_wrapper_multi</span></tt>(count, last_count, ...)</td>
|
|
<td>call the static method show_stat_wrapper for each process</td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">start</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">stop</span></tt>([make_sure_its_down])</td>
|
|
<td>trigger clean up by hand, needs to be done when not using</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</dd></dl>
|
|
|
|
<dl class="class">
|
|
<dt id="jobmanager.progress.ProgressBarCounter">
|
|
<em class="property">class </em><tt class="descclassname">jobmanager.progress.</tt><tt class="descname">ProgressBarCounter</tt><big>(</big><em>count</em>, <em>max_count=None</em>, <em>prepend=None</em>, <em>speed_calc_cycles_bar=10</em>, <em>speed_calc_cycles_counter=5</em>, <em>width='auto'</em>, <em>interval=1</em>, <em>verbose=0</em>, <em>sigint='stop'</em>, <em>sigterm='stop'</em>, <em>name='progress_bar_counter'</em><big>)</big><a class="reference internal" href="_modules/jobmanager/progress.html#ProgressBarCounter"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.ProgressBarCounter" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>records also the time of each reset and calculates the speed
|
|
of the resets.</p>
|
|
<p>shows the TET since init (not effected by reset)
|
|
the speed of the resets (number of finished processed per time)
|
|
and the number of finished processes</p>
|
|
<p>after that also show a progress of each process
|
|
max_count > 0 and not None -> bar
|
|
max_count == None -> absolute count statistic
|
|
max_count == 0 -> hide process statistic at all</p>
|
|
<p class="rubric">Attributes</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">run</span></tt></td>
|
|
<td>makes the shared memory boolean accessible as class attribute</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<p class="rubric">Methods</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">get_counter_count</span></tt>([i])</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">getpid</span></tt>()</td>
|
|
<td>return the process id of the spawned process</td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">is_alive</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">join</span></tt>(timeout)</td>
|
|
<td>calls join for the spawned process with given timeout</td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">pause</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">reset</span></tt>([i])</td>
|
|
<td>convenient function to reset progress information</td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">resume</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">show_stat</span></tt>(count_value, max_count_value, ...)</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">show_stat_wrapper</span></tt>(count, last_count, ...)</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">show_stat_wrapper_multi</span></tt>(count, last_count, ...)</td>
|
|
<td>call the static method show_stat_wrapper for each process</td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">start</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">stop</span></tt>([make_sure_its_down])</td>
|
|
<td>trigger clean up by hand, needs to be done when not using</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</dd></dl>
|
|
|
|
<dl class="class">
|
|
<dt id="jobmanager.progress.ProgressBarFancy">
|
|
<em class="property">class </em><tt class="descclassname">jobmanager.progress.</tt><tt class="descname">ProgressBarFancy</tt><big>(</big><em>count</em>, <em>max_count=None</em>, <em>width='auto'</em>, <em>prepend=None</em>, <em>speed_calc_cycles=10</em>, <em>interval=1</em>, <em>verbose=0</em>, <em>sigint='stop'</em>, <em>sigterm='stop'</em>, <em>name='progress_bar'</em><big>)</big><a class="reference internal" href="_modules/jobmanager/progress.html#ProgressBarFancy"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.ProgressBarFancy" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>implements a progress bar where the color indicates the current status
|
|
similar to the bars known from ‘htop’</p>
|
|
<p class="rubric">Attributes</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">run</span></tt></td>
|
|
<td>makes the shared memory boolean accessible as class attribute</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<p class="rubric">Methods</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">full_minor_stat</span></tt>(p, tet, speed, ttg, eta, ...)</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">full_stat</span></tt>(p, tet, speed, ttg, eta, ort, ...)</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">get_d</span></tt>(s1, s2, width, lp, lps)</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">getpid</span></tt>()</td>
|
|
<td>return the process id of the spawned process</td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">is_alive</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">join</span></tt>(timeout)</td>
|
|
<td>calls join for the spawned process with given timeout</td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">kw_bold</span></tt>(s, ch_after)</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">pause</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">reduced_1_stat</span></tt>(p, tet, speed, ttg, eta, ort, ...)</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">reduced_2_stat</span></tt>(p, tet, speed, ttg, eta, ort, ...)</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">reduced_3_stat</span></tt>(p, tet, speed, ttg, eta, ort, ...)</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">reduced_4_stat</span></tt>(p, tet, speed, ttg, eta, ort, ...)</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">reset</span></tt>([i])</td>
|
|
<td>convenient function to reset progress information</td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">resume</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">show_stat</span></tt>(count_value, max_count_value, ...)</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">show_stat_wrapper</span></tt>(count, last_count, ...)</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">show_stat_wrapper_multi</span></tt>(count, last_count, ...)</td>
|
|
<td>call the static method show_stat_wrapper for each process</td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">start</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">stop</span></tt>([make_sure_its_down])</td>
|
|
<td>trigger clean up by hand, needs to be done when not using</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</dd></dl>
|
|
|
|
<dl class="class">
|
|
<dt id="jobmanager.progress.SIG_handler_Loop">
|
|
<em class="property">class </em><tt class="descclassname">jobmanager.progress.</tt><tt class="descname">SIG_handler_Loop</tt><big>(</big><em>shared_mem_run</em>, <em>sigint</em>, <em>sigterm</em>, <em>identifier</em>, <em>verbose=0</em><big>)</big><a class="reference internal" href="_modules/jobmanager/progress.html#SIG_handler_Loop"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.SIG_handler_Loop" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>class to setup signal handling for the Loop class</p>
|
|
<p>Note: each subprocess receives the default signal handling from it’s parent.
|
|
If the signal function from the module signal is evoked within the subprocess
|
|
this default behavior can be overwritten.</p>
|
|
<p>The init function receives a shared memory boolean object which will be set
|
|
false in case of signal detection. Since the Loop class will check the state
|
|
of this boolean object before each repetition, the loop will stop when
|
|
a signal was receives.</p>
|
|
<p class="rubric">Methods</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">set_signal</span></tt>(sig, handler_str)</td>
|
|
<td></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</dd></dl>
|
|
|
|
<dl class="function">
|
|
<dt id="jobmanager.progress.get_terminal_size">
|
|
<tt class="descclassname">jobmanager.progress.</tt><tt class="descname">get_terminal_size</tt><big>(</big><em>defaultw=80</em><big>)</big><a class="reference internal" href="_modules/jobmanager/progress.html#get_terminal_size"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.get_terminal_size" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>Checks various methods to determine the terminal size</p>
|
|
<p>Methods:
|
|
- shutil.get_terminal_size (only Python3)
|
|
- fcntl.ioctl
|
|
- subprocess.check_output
|
|
- os.environ</p>
|
|
<table class="docutils field-list" frame="void" rules="none">
|
|
<col class="field-name" />
|
|
<col class="field-body" />
|
|
<tbody valign="top">
|
|
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><p class="first"><strong>defaultw</strong> : int</p>
|
|
<blockquote>
|
|
<div><p>Default width of terminal.</p>
|
|
</div></blockquote>
|
|
</td>
|
|
</tr>
|
|
<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><p class="first"><strong>width, height</strong> : int</p>
|
|
<blockquote class="last">
|
|
<div><p>Width and height of the terminal. If one of them could not be
|
|
found, None is return in its place.</p>
|
|
</div></blockquote>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</dd></dl>
|
|
|
|
<dl class="function">
|
|
<dt id="jobmanager.progress.humanize_speed">
|
|
<tt class="descclassname">jobmanager.progress.</tt><tt class="descname">humanize_speed</tt><big>(</big><em>c_per_sec</em><big>)</big><a class="reference internal" href="_modules/jobmanager/progress.html#humanize_speed"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.humanize_speed" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>convert a speed in counts per second to counts per [s, min, h, d], choosing the smallest value greater zero.</p>
|
|
</dd></dl>
|
|
|
|
<dl class="function">
|
|
<dt id="jobmanager.progress.humanize_time">
|
|
<tt class="descclassname">jobmanager.progress.</tt><tt class="descname">humanize_time</tt><big>(</big><em>secs</em><big>)</big><a class="reference internal" href="_modules/jobmanager/progress.html#humanize_time"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.humanize_time" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>convert second in to hh:mm:ss format</p>
|
|
</dd></dl>
|
|
|
|
<dl class="function">
|
|
<dt id="jobmanager.progress.terminal_reserve">
|
|
<tt class="descclassname">jobmanager.progress.</tt><tt class="descname">terminal_reserve</tt><big>(</big><em>progress_obj</em>, <em>terminal_obj=None</em>, <em>verbose=0</em>, <em>identifier=None</em><big>)</big><a class="reference internal" href="_modules/jobmanager/progress.html#terminal_reserve"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.terminal_reserve" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>Registers the terminal (stdout) for printing.</p>
|
|
<p>Useful to prevent multiple processes from writing progress bars
|
|
to stdout.</p>
|
|
<p>One process (server) prints to stdout and a couple of subprocesses
|
|
do not print to the same stdout, because the server has reserved it.
|
|
Of course, the clients have to be nice and check with
|
|
terminal_reserve first if they should (not) print.
|
|
Nothing is locked.</p>
|
|
<table class="docutils field-list" frame="void" rules="none">
|
|
<col class="field-name" />
|
|
<col class="field-body" />
|
|
<tbody valign="top">
|
|
<tr class="field-odd field"><th class="field-name">Returns:</th><td class="field-body"><p class="first">True if reservation was successful (or if we have already reserved this tty),</p>
|
|
<p class="last">False if there already is a reservation from another instance.</p>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</dd></dl>
|
|
|
|
<dl class="function">
|
|
<dt id="jobmanager.progress.terminal_unreserve">
|
|
<tt class="descclassname">jobmanager.progress.</tt><tt class="descname">terminal_unreserve</tt><big>(</big><em>progress_obj</em>, <em>terminal_obj=None</em>, <em>verbose=0</em>, <em>identifier=None</em><big>)</big><a class="reference internal" href="_modules/jobmanager/progress.html#terminal_unreserve"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.progress.terminal_unreserve" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>Unregisters the terminal (stdout) for printing.</p>
|
|
<p>an instance (progress_obj) can only unreserve the tty (terminal_obj) when it also reserved it</p>
|
|
<p>see terminal_reserved for more information</p>
|
|
<table class="docutils field-list" frame="void" rules="none">
|
|
<col class="field-name" />
|
|
<col class="field-body" />
|
|
<tbody valign="top">
|
|
<tr class="field-odd field"><th class="field-name">Returns:</th><td class="field-body">None</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</dd></dl>
|
|
|
|
</div>
|
|
<div class="section" id="user-convenience">
|
|
<h2>User Convenience<a class="headerlink" href="#user-convenience" title="Permalink to this headline">¶</a></h2>
|
|
<div class="section" id="module-jobmanager.decorators">
|
|
<span id="decorators"></span><h3>Decorators<a class="headerlink" href="#module-jobmanager.decorators" title="Permalink to this headline">¶</a></h3>
|
|
<p>Implements decorators/wrappers for simple use-cases of jobmanager.</p>
|
|
<dl class="class">
|
|
<dt id="jobmanager.decorators.ProgressBar">
|
|
<em class="property">class </em><tt class="descclassname">jobmanager.decorators.</tt><tt class="descname">ProgressBar</tt><big>(</big><em>func</em>, <em>**kwargs</em><big>)</big><a class="reference internal" href="_modules/jobmanager/decorators.html#ProgressBar"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.decorators.ProgressBar" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>A wrapper/decorator with a text-based progress bar.</p>
|
|
<p>Methods:
|
|
- __init__
|
|
- __call__</p>
|
|
<p>The idea is to add a status bar for a regular
|
|
function just by wrapping the function via
|
|
python’s decorator syntax.</p>
|
|
<p>In order to do so, the function needs to provide some
|
|
extra information, namely the current state ‘count’ and
|
|
the final state ‘max_count’. Simply expand your function
|
|
by these two additional keyword arguments (or other pairs
|
|
specified in jobmanager.validCountKwargs) and set their
|
|
values during the calculation (see example 1 below). In that
|
|
manner the decorated function as well as the not decorated
|
|
function can simple be called as one would not care about
|
|
any status information.</p>
|
|
<p>Alternatively one could explicitly set count and max_count
|
|
in the function call, which circumvents the need to change
|
|
the value of max_count AFTER instantiation of the progressBar.</p>
|
|
<p class="rubric">Notes</p>
|
|
<p>You can also use this class as a wrapper and tune parameters of the
|
|
progress bar.</p>
|
|
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="n">wrapper</span> <span class="o">=</span> <span class="n">ProgressBar</span><span class="p">(</span><span class="n">my_func</span><span class="p">,</span> <span class="n">interval</span><span class="o">=.</span><span class="mi">1</span><span class="p">)</span>
|
|
<span class="gp">>>> </span><span class="n">result</span> <span class="o">=</span> <span class="n">wrapper</span><span class="p">(</span><span class="s">"wrapped function"</span><span class="p">,</span> <span class="n">kwarg</span><span class="o">=</span><span class="s">" test"</span><span class="p">)</span>
|
|
</pre></div>
|
|
</div>
|
|
<p class="rubric">Methods</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">__call__</span></tt>(*args, **kwargs)</td>
|
|
<td>Calls <cite>func</cite> - previously defined in <cite>__init__</cite>.</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</dd></dl>
|
|
|
|
<dl class="class">
|
|
<dt id="jobmanager.decorators.ProgressBarExtended">
|
|
<em class="property">class </em><tt class="descclassname">jobmanager.decorators.</tt><tt class="descname">ProgressBarExtended</tt><big>(</big><em>func</em>, <em>**kwargs</em><big>)</big><a class="reference internal" href="_modules/jobmanager/decorators.html#ProgressBarExtended"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.decorators.ProgressBarExtended" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>extends the ProgressBar such that</p>
|
|
<p>on can turn of the ProgressBar by giving an extra argument,
|
|
namely ‘progress_bar_off’ and set its value to ‘True’.</p>
|
|
<p>further there will be an additional argument passed to the function
|
|
called ‘progress_bar’ which allows to stop the progress bar from
|
|
within the function. note that there will be an function signature error
|
|
if the function does not accept the extra argument ‘progress_bar’. So a
|
|
general <a href="#id3"><span class="problematic" id="id4">**</span></a>kwargs at the end of the functions arguments will help.
|
|
That is also the reason why the extended version comes in an extra class
|
|
because it might otherwise break compatibility.</p>
|
|
<p class="rubric">Methods</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">__call__</span></tt>(*args, **kwargs)</td>
|
|
<td></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</dd></dl>
|
|
|
|
<dl class="function">
|
|
<dt id="jobmanager.decorators.decorate_module_ProgressBar">
|
|
<tt class="descclassname">jobmanager.decorators.</tt><tt class="descname">decorate_module_ProgressBar</tt><big>(</big><em>module</em>, <em>decorator=<class 'jobmanager.decorators.ProgressBar'></em>, <em>**kwargs</em><big>)</big><a class="reference internal" href="_modules/jobmanager/decorators.html#decorate_module_ProgressBar"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.decorators.decorate_module_ProgressBar" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>Decorates all decoratable functions in a module with a
|
|
ProgressBar.</p>
|
|
<p>You can prevent wrapping of a function by not specifying the keyword
|
|
arguments as defined in <cite>jobmanager.jobmanager.validCountKwargs</cite> or
|
|
by defining a function <a href="#id5"><span class="problematic" id="id6">`</span></a>_jm_decorate_{func}”.</p>
|
|
<table class="docutils field-list" frame="void" rules="none">
|
|
<col class="field-name" />
|
|
<col class="field-body" />
|
|
<tbody valign="top">
|
|
<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><p class="first"><strong>module</strong> : Python module</p>
|
|
<blockquote>
|
|
<div><p>The module whose functions should be decorated.</p>
|
|
</div></blockquote>
|
|
<p><strong>decorator</strong> : bool</p>
|
|
<blockquote>
|
|
<div><p>Specifies a decorator in jobmanager.decorators that should be
|
|
used.</p>
|
|
</div></blockquote>
|
|
<p><strong>**kwargs</strong> : dict</p>
|
|
<blockquote class="last">
|
|
<div><p>Keyword arguments to the ProgressBar.</p>
|
|
</div></blockquote>
|
|
</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<p class="rubric">Notes</p>
|
|
<p>Decorating nested functions in a module might lead to unexpected
|
|
behavior.</p>
|
|
</dd></dl>
|
|
|
|
</div>
|
|
<div class="section" id="module-jobmanager.ode_wrapper">
|
|
<span id="ode-wrapper"></span><h3>ODE wrapper<a class="headerlink" href="#module-jobmanager.ode_wrapper" title="Permalink to this headline">¶</a></h3>
|
|
<dl class="function">
|
|
<dt id="jobmanager.ode_wrapper.wrap_complex_intgeration">
|
|
<tt class="descclassname">jobmanager.ode_wrapper.</tt><tt class="descname">wrap_complex_intgeration</tt><big>(</big><em>f_complex</em><big>)</big><a class="reference internal" href="_modules/jobmanager/ode_wrapper.html#wrap_complex_intgeration"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.ode_wrapper.wrap_complex_intgeration" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>if f: R x C^n -> C^n
|
|
then this functions returns the real equivalent
|
|
f_prime R x R^n x R^n -> R^n x R^n</p>
|
|
<dl class="docutils">
|
|
<dt>such that a complex vector</dt>
|
|
<dd>cc = [vc_1, ... vc_n]</dd>
|
|
<dt>translates to</dt>
|
|
<dd>cr = [RE(vc_1), ... RE(vc_n), IM(vc_1), ... IM(vc_n)]</dd>
|
|
</dl>
|
|
</dd></dl>
|
|
|
|
</div>
|
|
</div>
|
|
<div class="section" id="core-modules">
|
|
<h2>Core modules<a class="headerlink" href="#core-modules" title="Permalink to this headline">¶</a></h2>
|
|
<div class="section" id="jobmanager">
|
|
<h3>jobmanager<a class="headerlink" href="#jobmanager" title="Permalink to this headline">¶</a></h3>
|
|
</div>
|
|
<div class="section" id="module-jobmanager.jobmanager">
|
|
<span id="clients"></span><h3>clients<a class="headerlink" href="#module-jobmanager.jobmanager" title="Permalink to this headline">¶</a></h3>
|
|
<p>jobmanager module</p>
|
|
<p>Richard Hartmann 2014</p>
|
|
<p>This module provides an easy way to implement distributed computing
|
|
based on the python class SyncManager for remote communication
|
|
and the python module multiprocessing for local parallelism.</p>
|
|
<p>class SIG_handler_Loop</p>
|
|
<p>The class Loop provides as mechanism to spawn a process repeating to
|
|
call a certain function as well as a StatusBar class for the terminal.</p>
|
|
<p>class StatusBar</p>
|
|
<p>The class JobManager_Server will provide a server process handling the
|
|
following tasks:</p>
|
|
<blockquote>
|
|
<div><ul class="simple">
|
|
<li>providing a list (queue) of arguments to be processed by client processes</li>
|
|
</ul>
|
|
<p>(see put_arg and args_from_list)
|
|
- handling the results of the calculations done by the client processes
|
|
(see process_new_result)
|
|
- when finished (all provided arguments have been processed and returned its result)
|
|
process the obtained results (see process_final_result)</p>
|
|
</div></blockquote>
|
|
<p>The class JobManager_Client</p>
|
|
<dl class="class">
|
|
<dt id="jobmanager.jobmanager.JobManager_Client">
|
|
<em class="property">class </em><tt class="descclassname">jobmanager.jobmanager.</tt><tt class="descname">JobManager_Client</tt><big>(</big><em>server</em>, <em>authkey</em>, <em>port=42524</em>, <em>nproc=0</em>, <em>njobs=0</em>, <em>nice=19</em>, <em>no_warnings=False</em>, <em>verbose=1</em>, <em>show_statusbar_for_jobs=True</em>, <em>show_counter_only=False</em>, <em>interval=0.3</em><big>)</big><a class="reference internal" href="_modules/jobmanager/jobmanager.html#JobManager_Client"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.jobmanager.JobManager_Client" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>Calls the functions self.func with arguments fetched from the job_q.
|
|
You should subclass this class and overwrite func to handle your own
|
|
function.</p>
|
|
<p>The job_q is provided by the SyncManager who connects to a
|
|
SyncManager setup by the JobManager_Server.</p>
|
|
<p>Spawns nproc subprocesses (__worker_func) to process arguments.
|
|
Each subprocess gets an argument from the job_q, processes it
|
|
and puts the result to the result_q.</p>
|
|
<p>If the job_q is empty, terminate the subprocess.</p>
|
|
<p>In case of any failure detected within the try except clause
|
|
the argument, which just failed to process, the error and the
|
|
hostname are put to the fail_q so the JobManager_Server can take
|
|
care of that.</p>
|
|
<p>After that the traceback is written to a file with name
|
|
traceback_args_<args>_err_<err>_<YYYY>_<MM>_<DD>_<hh>_<mm>_<ss>_<PID>.trb.</p>
|
|
<p>Then the process will terminate.</p>
|
|
<p class="rubric">Attributes</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">connected</span></tt></td>
|
|
<td></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<p class="rubric">Methods</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">connect</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><a class="reference internal" href="#jobmanager.jobmanager.JobManager_Client.func" title="jobmanager.jobmanager.JobManager_Client.func"><tt class="xref py py-obj docutils literal"><span class="pre">func</span></tt></a>(arg, const_arg)</td>
|
|
<td>function to be called by the worker processes</td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">get_manager_objects</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><a class="reference internal" href="#jobmanager.jobmanager.JobManager_Client.start" title="jobmanager.jobmanager.JobManager_Client.start"><tt class="xref py py-obj docutils literal"><span class="pre">start</span></tt></a>()</td>
|
|
<td>starts a number of nproc subprocess to work on the job_q</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<dl class="staticmethod">
|
|
<dt id="jobmanager.jobmanager.JobManager_Client.func">
|
|
<em class="property">static </em><tt class="descname">func</tt><big>(</big><em>arg</em>, <em>const_arg</em><big>)</big><a class="reference internal" href="_modules/jobmanager/jobmanager.html#JobManager_Client.func"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.jobmanager.JobManager_Client.func" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>function to be called by the worker processes</p>
|
|
<p>arg - provided by the job_q of the JobManager_Server</p>
|
|
<p>const_arg - tuple of constant arguments also provided by the JobManager_Server</p>
|
|
<p>to give status information to the Client class, use the variables
|
|
(c, m) as additional parameters. c and m will be
|
|
multiprocessing.sharedctypes.Synchronized objects with an underlying
|
|
unsigned int. so set c.value to the current status of the operation
|
|
ans m.value to the final status. So at the end of the operation c.value should
|
|
be m.value.</p>
|
|
<p>NOTE: This is just some dummy implementation to be used for test reasons only!
|
|
Subclass and overwrite this function to implement your own function.</p>
|
|
</dd></dl>
|
|
|
|
<dl class="method">
|
|
<dt id="jobmanager.jobmanager.JobManager_Client.start">
|
|
<tt class="descname">start</tt><big>(</big><big>)</big><a class="reference internal" href="_modules/jobmanager/jobmanager.html#JobManager_Client.start"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.jobmanager.JobManager_Client.start" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>starts a number of nproc subprocess to work on the job_q</p>
|
|
<p>SIGTERM and SIGINT are managed to terminate all subprocesses</p>
|
|
<p>retruns when all subprocesses have terminated</p>
|
|
</dd></dl>
|
|
|
|
</dd></dl>
|
|
|
|
<dl class="class">
|
|
<dt id="jobmanager.jobmanager.JobManager_Server">
|
|
<em class="property">class </em><tt class="descclassname">jobmanager.jobmanager.</tt><tt class="descname">JobManager_Server</tt><big>(</big><em>authkey</em>, <em>const_arg=None</em>, <em>port=42524</em>, <em>verbose=1</em>, <em>msg_interval=1</em>, <em>fname_dump='auto'</em>, <em>speed_calc_cycles=50</em><big>)</big><a class="reference internal" href="_modules/jobmanager/jobmanager.html#JobManager_Server"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.jobmanager.JobManager_Server" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>general usage:</p>
|
|
<blockquote>
|
|
<div><ul class="simple">
|
|
<li>init the JobManager_Server, start SyncManager server process</li>
|
|
<li>pass the arguments to be processed to the JobManager_Server</li>
|
|
</ul>
|
|
<p>(put_arg, args_from_list)</p>
|
|
<ul class="simple">
|
|
<li>start the JobManager_Server (start), which means to wait for incoming</li>
|
|
</ul>
|
|
<p>results and to process them. Afterwards process all obtained data.</p>
|
|
</div></blockquote>
|
|
<p>The default behavior of handling each incoming new result is to simply
|
|
add the pair (arg, result) to the final_result list.</p>
|
|
<p>When finished the default final processing is to dump the
|
|
final_result list to fname_for_final_result_dump</p>
|
|
<p>To change this behavior you may subclass the JobManager_Server
|
|
and implement</p>
|
|
<blockquote>
|
|
<div><ul class="simple">
|
|
<li>an extended __init__ to change the type of the final_result attribute</li>
|
|
<li>process_new_result</li>
|
|
<li>process_final_result(self)</li>
|
|
</ul>
|
|
</div></blockquote>
|
|
<p>In case of any exceptions the JobManager_Server will call process_final_result
|
|
and dump the unprocessed job_q as a list to fname_for_job_q_dump.</p>
|
|
<p>Also the signal SIGTERM is caught. In such a case it will raise SystemExit exception
|
|
will will then be handle in the try except clause.</p>
|
|
<p>SystemExit and KeyboardInterrupt exceptions are not considered as failure. They are
|
|
rather methods to shut down the Server gracefully. Therefore in such cases no
|
|
traceback will be printed.</p>
|
|
<p>All other exceptions are probably due to some failure in function. A traceback
|
|
it printed to stderr.</p>
|
|
<dl class="docutils">
|
|
<dt>notes:</dt>
|
|
<dd><ul class="first simple">
|
|
<li>when the JobManager_Server gets killed (SIGKILL) and the SyncManager still</li>
|
|
</ul>
|
|
<p>lives, the port used will occupied. considere sudo natstat -pna | grep 42524 to
|
|
find the process still using the port</p>
|
|
<ul class="simple">
|
|
<li>also the SyncManager ignores SIGTERM and SIGINT signals so you have to send</li>
|
|
</ul>
|
|
<p class="last">a SIGKILL.</p>
|
|
</dd>
|
|
</dl>
|
|
<p class="rubric">Attributes</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">numjobs</span></tt></td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">numresults</span></tt></td>
|
|
<td></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<p class="rubric">Methods</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><a class="reference internal" href="#jobmanager.jobmanager.JobManager_Server.args_from_list" title="jobmanager.jobmanager.JobManager_Server.args_from_list"><tt class="xref py py-obj docutils literal"><span class="pre">args_from_list</span></tt></a>(args)</td>
|
|
<td>serialize a list of arguments to the job_q</td>
|
|
</tr>
|
|
<tr class="row-even"><td><a class="reference internal" href="#jobmanager.jobmanager.JobManager_Server.process_final_result" title="jobmanager.jobmanager.JobManager_Server.process_final_result"><tt class="xref py py-obj docutils literal"><span class="pre">process_final_result</span></tt></a>()</td>
|
|
<td>to implement user defined final processing</td>
|
|
</tr>
|
|
<tr class="row-odd"><td><a class="reference internal" href="#jobmanager.jobmanager.JobManager_Server.process_new_result" title="jobmanager.jobmanager.JobManager_Server.process_new_result"><tt class="xref py py-obj docutils literal"><span class="pre">process_new_result</span></tt></a>(arg, result)</td>
|
|
<td>Will be called when the result_q has data available.</td>
|
|
</tr>
|
|
<tr class="row-even"><td><a class="reference internal" href="#jobmanager.jobmanager.JobManager_Server.put_arg" title="jobmanager.jobmanager.JobManager_Server.put_arg"><tt class="xref py py-obj docutils literal"><span class="pre">put_arg</span></tt></a>(a)</td>
|
|
<td>add argument a to the job_q</td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">read_old_state</span></tt>([fname_dump])</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">show_statistics</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><a class="reference internal" href="#jobmanager.jobmanager.JobManager_Server.shutdown" title="jobmanager.jobmanager.JobManager_Server.shutdown"><tt class="xref py py-obj docutils literal"><span class="pre">shutdown</span></tt></a>()</td>
|
|
<td>“stop all spawned processes and clean up</td>
|
|
</tr>
|
|
<tr class="row-even"><td><a class="reference internal" href="#jobmanager.jobmanager.JobManager_Server.start" title="jobmanager.jobmanager.JobManager_Server.start"><tt class="xref py py-obj docutils literal"><span class="pre">start</span></tt></a>()</td>
|
|
<td>starts to loop over incoming results</td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">static_load</span></tt>(f)</td>
|
|
<td></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<dl class="method">
|
|
<dt id="jobmanager.jobmanager.JobManager_Server.args_from_list">
|
|
<tt class="descname">args_from_list</tt><big>(</big><em>args</em><big>)</big><a class="reference internal" href="_modules/jobmanager/jobmanager.html#JobManager_Server.args_from_list"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.jobmanager.JobManager_Server.args_from_list" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>serialize a list of arguments to the job_q</p>
|
|
</dd></dl>
|
|
|
|
<dl class="method">
|
|
<dt id="jobmanager.jobmanager.JobManager_Server.process_final_result">
|
|
<tt class="descname">process_final_result</tt><big>(</big><big>)</big><a class="reference internal" href="_modules/jobmanager/jobmanager.html#JobManager_Server.process_final_result"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.jobmanager.JobManager_Server.process_final_result" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>to implement user defined final processing</p>
|
|
</dd></dl>
|
|
|
|
<dl class="method">
|
|
<dt id="jobmanager.jobmanager.JobManager_Server.process_new_result">
|
|
<tt class="descname">process_new_result</tt><big>(</big><em>arg</em>, <em>result</em><big>)</big><a class="reference internal" href="_modules/jobmanager/jobmanager.html#JobManager_Server.process_new_result"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.jobmanager.JobManager_Server.process_new_result" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>Will be called when the result_q has data available.
|
|
result is the computed result to the argument arg.</p>
|
|
<p>Should be overwritten by subclassing!</p>
|
|
</dd></dl>
|
|
|
|
<dl class="method">
|
|
<dt id="jobmanager.jobmanager.JobManager_Server.put_arg">
|
|
<tt class="descname">put_arg</tt><big>(</big><em>a</em><big>)</big><a class="reference internal" href="_modules/jobmanager/jobmanager.html#JobManager_Server.put_arg"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.jobmanager.JobManager_Server.put_arg" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>add argument a to the job_q</p>
|
|
</dd></dl>
|
|
|
|
<dl class="method">
|
|
<dt id="jobmanager.jobmanager.JobManager_Server.shutdown">
|
|
<tt class="descname">shutdown</tt><big>(</big><big>)</big><a class="reference internal" href="_modules/jobmanager/jobmanager.html#JobManager_Server.shutdown"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.jobmanager.JobManager_Server.shutdown" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>“stop all spawned processes and clean up</p>
|
|
<ul class="simple">
|
|
<li>call process_final_result to handle all collected result</li>
|
|
<li>if job_q is not empty dump remaining job_q</li>
|
|
</ul>
|
|
</dd></dl>
|
|
|
|
<dl class="method">
|
|
<dt id="jobmanager.jobmanager.JobManager_Server.start">
|
|
<tt class="descname">start</tt><big>(</big><big>)</big><a class="reference internal" href="_modules/jobmanager/jobmanager.html#JobManager_Server.start"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.jobmanager.JobManager_Server.start" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>starts to loop over incoming results</p>
|
|
<p>When finished, or on exception call stop() afterwards to shut down gracefully.</p>
|
|
</dd></dl>
|
|
|
|
</dd></dl>
|
|
|
|
<dl class="function">
|
|
<dt id="jobmanager.jobmanager.getDateForFileName">
|
|
<tt class="descclassname">jobmanager.jobmanager.</tt><tt class="descname">getDateForFileName</tt><big>(</big><em>includePID=False</em><big>)</big><a class="reference internal" href="_modules/jobmanager/jobmanager.html#getDateForFileName"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.jobmanager.getDateForFileName" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>returns the current date-time and optionally the process id in the format
|
|
YYYY_MM_DD_hh_mm_ss_pid</p>
|
|
</dd></dl>
|
|
|
|
</div>
|
|
<div class="section" id="id7">
|
|
<h3>clients<a class="headerlink" href="#id7" title="Permalink to this headline">¶</a></h3>
|
|
<span class="target" id="module-jobmanager.clients"></span><p>The clients module</p>
|
|
<p>This module provides special subclasses of the JobManager_Client</p>
|
|
<dl class="class">
|
|
<dt id="jobmanager.clients.Integration_Client_CPLX">
|
|
<em class="property">class </em><tt class="descclassname">jobmanager.clients.</tt><tt class="descname">Integration_Client_CPLX</tt><big>(</big><em>**kwargs</em><big>)</big><a class="reference internal" href="_modules/jobmanager/clients.html#Integration_Client_CPLX"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.clients.Integration_Client_CPLX" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>A JobManager_Client subclass to integrate a set of complex valued ODE.</p>
|
|
<p>‘arg’ and ‘const_arg’ are understood as keyword arguments in oder to
|
|
call ode_wrapper.integrate_cplx. They are passed to merge_arg_and_const_arg
|
|
in order to separate the kwargs needed by ode_wrapper.integrate_cplx
|
|
from the args (as tupled) passed to the function calculating derivative of the DGL.
|
|
This tuple of parameters itself is passed as a special argument to
|
|
ode_wrapper.integrate_cplx namely ‘args’.</p>
|
|
<p>If ‘arg’ or ‘const_arg’ provide the attribute ‘_asdict’ it will be called
|
|
in order to construct dictionaries and use them for further processing.</p>
|
|
<p>The following keys MUST be provided by ‘arg’ or ‘const_arg’</p>
|
|
<blockquote>
|
|
<div><p>t0 : initial time
|
|
t1 : final time
|
|
N : number of time steps for the solution x(t)</p>
|
|
<blockquote>
|
|
<div>t = linspace(t0, t1, N)</div></blockquote>
|
|
<p>f : function holding the derivatives
|
|
args : additional positional arguments passed to f(t, x, <a href="#id8"><span class="problematic" id="id9">*</span></a>args)
|
|
x0 : initial value
|
|
integrator : type of integration method</p>
|
|
<blockquote>
|
|
<div><dl class="docutils">
|
|
<dt>‘zvode’: complex version of vode,</dt>
|
|
<dd>in case of stiff ode, f needs to be analytic
|
|
see also scipy.integrate.ode -> ‘zvode’
|
|
most efficient for complex ODE</dd>
|
|
<dt>‘vode’, ‘lsoda’: both do automatic converkwargs.pop(‘args’)sion from the</dt>
|
|
<dd>complex ODE to the doubly dimensioned real
|
|
system of ODE, and use the corresponding real
|
|
integrator methods.
|
|
Might be of the order of one magnitude slower
|
|
that ‘zvode’. Consider using Integration_Client_REAL
|
|
in the first place.</dd>
|
|
</dl>
|
|
</div></blockquote>
|
|
</div></blockquote>
|
|
<dl class="docutils">
|
|
<dt>optional keys are:</dt>
|
|
<dd>verbose : default 0
|
|
integrator related arguments (see the scipy doc ODE)</dd>
|
|
</dl>
|
|
<p>The key ‘args’ itself (should be tuple) will be merged as
|
|
kwargs[‘args’] = arg[‘args’] + const_arg[‘args’]
|
|
which means that the call signature of f has to be
|
|
f(t, x, arg_1, arg_2, ... const_arg_1, const_arg_2, ...).</p>
|
|
<p class="rubric">Attributes</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">connected</span></tt></td>
|
|
<td></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<p class="rubric">Methods</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">connect</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">func</span></tt>(arg, const_arg, c, m)</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">get_manager_objects</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">start</span></tt>()</td>
|
|
<td>starts a number of nproc subprocess to work on the job_q</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</dd></dl>
|
|
|
|
<dl class="class">
|
|
<dt id="jobmanager.clients.Integration_Client_REAL">
|
|
<em class="property">class </em><tt class="descclassname">jobmanager.clients.</tt><tt class="descname">Integration_Client_REAL</tt><big>(</big><em>**kwargs</em><big>)</big><a class="reference internal" href="_modules/jobmanager/clients.html#Integration_Client_REAL"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.clients.Integration_Client_REAL" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>A JobManager_Client subclass to integrate a set of complex real ODE.</p>
|
|
<p>same behavior as described for Integration_Client_CPLX except
|
|
that ‘vode’ and ‘lsoda’ do not do any wrapping, so there is no
|
|
performance issue and ‘zvode’ is obviously not supported.</p>
|
|
<p class="rubric">Attributes</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">connected</span></tt></td>
|
|
<td></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<p class="rubric">Methods</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">connect</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">func</span></tt>(arg, const_arg, c, m)</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">get_manager_objects</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">start</span></tt>()</td>
|
|
<td>starts a number of nproc subprocess to work on the job_q</td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
</dd></dl>
|
|
|
|
<dl class="function">
|
|
<dt id="jobmanager.clients.merge_arg_and_const_arg">
|
|
<tt class="descclassname">jobmanager.clients.</tt><tt class="descname">merge_arg_and_const_arg</tt><big>(</big><em>arg</em>, <em>const_arg</em><big>)</big><a class="reference internal" href="_modules/jobmanager/clients.html#merge_arg_and_const_arg"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.clients.merge_arg_and_const_arg" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>prepares data from arg and const_arg such that they can be passed
|
|
to the general integration routine</p>
|
|
<p>arg and const_arg are both assumed to be dictionaries</p>
|
|
<p>the merge process must not alter arg nor const_arg
|
|
in order to be used in the jobmanager context</p>
|
|
<p>returns the arguments passed to the function
|
|
defining the derivative such that
|
|
args_dgl = arg[‘args’] + const_arg[‘args’]
|
|
where as arg[‘args’] and const_arg[‘args’] have been assumed to be tuples</p>
|
|
<dl class="docutils">
|
|
<dt>e.g. </dt>
|
|
<dd>arg[‘args’] = (2, ‘low’)
|
|
const_arg[‘args’] = (15, np.pi)</dd>
|
|
</dl>
|
|
<p>f will be called with
|
|
f(t, x, 2, ‘low’, 15, np.pi)</p>
|
|
<p>returns further the combined dictionary
|
|
arg + const_arg with the keyword ‘args’ removed</p>
|
|
<p>For any duplicate keys the value will be the value
|
|
from the ‘arg’ dictionary.</p>
|
|
</dd></dl>
|
|
|
|
</div>
|
|
<div class="section" id="module-jobmanager.servers">
|
|
<span id="servers"></span><h3>servers<a class="headerlink" href="#module-jobmanager.servers" title="Permalink to this headline">¶</a></h3>
|
|
<dl class="function">
|
|
<dt id="jobmanager.servers.recursive_scan_for_instance">
|
|
<tt class="descclassname">jobmanager.servers.</tt><tt class="descname">recursive_scan_for_instance</tt><big>(</big><em>obj</em>, <em>type</em>, <em>explicit_exclude=None</em><big>)</big><a class="reference internal" href="_modules/jobmanager/servers.html#recursive_scan_for_instance"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.servers.recursive_scan_for_instance" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>try to do some recursive check to see whether ‘obj’ is of type
|
|
‘type’ or contains items of ‘type’ type.</p>
|
|
<p>if obj is a mapping (like dict) this will only check
|
|
for item iterated over via</p>
|
|
<blockquote>
|
|
<div>for item in obj</div></blockquote>
|
|
<p>which corresponds to the keys in the dict case.</p>
|
|
<p>The explicit_exclude argument may be a tuple of types for
|
|
some explicit checking in the sense that if obj is an
|
|
instance of one of the type given by explicit_exclude
|
|
we know it is NOT an instance of type.</p>
|
|
</dd></dl>
|
|
|
|
</div>
|
|
<div class="section" id="module-jobmanager.persistentData">
|
|
<span id="persistentdata"></span><h3>persistentData<a class="headerlink" href="#module-jobmanager.persistentData" title="Permalink to this headline">¶</a></h3>
|
|
<dl class="class">
|
|
<dt id="jobmanager.persistentData.PersistentDataStructure">
|
|
<em class="property">class </em><tt class="descclassname">jobmanager.persistentData.</tt><tt class="descname">PersistentDataStructure</tt><big>(</big><em>name</em>, <em>path='./'</em>, <em>verbose=1</em><big>)</big><a class="reference internal" href="_modules/jobmanager/persistentData.html#PersistentDataStructure"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.persistentData.PersistentDataStructure" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>Note: avoid using pickled dictionaries as binary keys! The problem with dicts is
|
|
that the order of the keys, when returned as list, depends on the hash value of
|
|
the keys. If the keys are strings, the hash value will be randomly seeded for
|
|
each python session, which may lead to different binary representations of the
|
|
same dict. Therefore the same dict may actually be considered as distinct keys.</p>
|
|
<p>The same hold true when using classes with default pickler routine as binary keys
|
|
(because the pickler will essentially pickle the dictionary self.__dict__).
|
|
If you want to use “complicated” python objects as binary keys make sure you
|
|
implement your own pickle behavior without the need of dictionaries.</p>
|
|
<p class="rubric">Methods</p>
|
|
<table border="1" class="longtable docutils">
|
|
<colgroup>
|
|
<col width="10%" />
|
|
<col width="90%" />
|
|
</colgroup>
|
|
<tbody valign="top">
|
|
<tr class="row-odd"><td><a class="reference internal" href="#jobmanager.persistentData.PersistentDataStructure.clear" title="jobmanager.persistentData.PersistentDataStructure.clear"><tt class="xref py py-obj docutils literal"><span class="pre">clear</span></tt></a>()</td>
|
|
<td>delete all entries from the db</td>
|
|
</tr>
|
|
<tr class="row-even"><td><a class="reference internal" href="#jobmanager.persistentData.PersistentDataStructure.close" title="jobmanager.persistentData.PersistentDataStructure.close"><tt class="xref py py-obj docutils literal"><span class="pre">close</span></tt></a>()</td>
|
|
<td>close the sqligtedict ans therefore the SQL database</td>
|
|
</tr>
|
|
<tr class="row-odd"><td><a class="reference internal" href="#jobmanager.persistentData.PersistentDataStructure.erase" title="jobmanager.persistentData.PersistentDataStructure.erase"><tt class="xref py py-obj docutils literal"><span class="pre">erase</span></tt></a>()</td>
|
|
<td>removed the database file from the disk</td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">getData</span></tt>(key[, create_sub_data])</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">has_key</span></tt>(key)</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">is_closed</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">is_open</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">is_subdata</span></tt>(key)</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-odd"><td><tt class="xref py py-obj docutils literal"><span class="pre">need_open</span></tt>()</td>
|
|
<td></td>
|
|
</tr>
|
|
<tr class="row-even"><td><a class="reference internal" href="#jobmanager.persistentData.PersistentDataStructure.newSubData" title="jobmanager.persistentData.PersistentDataStructure.newSubData"><tt class="xref py py-obj docutils literal"><span class="pre">newSubData</span></tt></a>(key)</td>
|
|
<td>if key is not in database</td>
|
|
</tr>
|
|
<tr class="row-odd"><td><a class="reference internal" href="#jobmanager.persistentData.PersistentDataStructure.open" title="jobmanager.persistentData.PersistentDataStructure.open"><tt class="xref py py-obj docutils literal"><span class="pre">open</span></tt></a>()</td>
|
|
<td>open the SQL database at self._filename = <path>/__<name>/<name>.db</td>
|
|
</tr>
|
|
<tr class="row-even"><td><a class="reference internal" href="#jobmanager.persistentData.PersistentDataStructure.setData" title="jobmanager.persistentData.PersistentDataStructure.setData"><tt class="xref py py-obj docutils literal"><span class="pre">setData</span></tt></a>(key, value[, overwrite])</td>
|
|
<td>write the key value pair to the data base</td>
|
|
</tr>
|
|
<tr class="row-odd"><td><a class="reference internal" href="#jobmanager.persistentData.PersistentDataStructure.setDataFromSubData" title="jobmanager.persistentData.PersistentDataStructure.setDataFromSubData"><tt class="xref py py-obj docutils literal"><span class="pre">setDataFromSubData</span></tt></a>(key, subData)</td>
|
|
<td>set an entry of the PDS with data from an other PDS</td>
|
|
</tr>
|
|
<tr class="row-even"><td><tt class="xref py py-obj docutils literal"><span class="pre">show_stat</span></tt>([recursive, prepend])</td>
|
|
<td></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<dl class="method">
|
|
<dt id="jobmanager.persistentData.PersistentDataStructure.clear">
|
|
<tt class="descname">clear</tt><big>(</big><big>)</big><a class="reference internal" href="_modules/jobmanager/persistentData.html#PersistentDataStructure.clear"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.persistentData.PersistentDataStructure.clear" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>delete all entries from the db</p>
|
|
</dd></dl>
|
|
|
|
<dl class="method">
|
|
<dt id="jobmanager.persistentData.PersistentDataStructure.close">
|
|
<tt class="descname">close</tt><big>(</big><big>)</big><a class="reference internal" href="_modules/jobmanager/persistentData.html#PersistentDataStructure.close"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.persistentData.PersistentDataStructure.close" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>close the sqligtedict ans therefore the SQL database</p>
|
|
</dd></dl>
|
|
|
|
<dl class="method">
|
|
<dt id="jobmanager.persistentData.PersistentDataStructure.erase">
|
|
<tt class="descname">erase</tt><big>(</big><big>)</big><a class="reference internal" href="_modules/jobmanager/persistentData.html#PersistentDataStructure.erase"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.persistentData.PersistentDataStructure.erase" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>removed the database file from the disk</p>
|
|
<p>this is called recursively for all sub PersistentDataStructure</p>
|
|
</dd></dl>
|
|
|
|
<dl class="method">
|
|
<dt id="jobmanager.persistentData.PersistentDataStructure.newSubData">
|
|
<tt class="descname">newSubData</tt><big>(</big><em>key</em><big>)</big><a class="reference internal" href="_modules/jobmanager/persistentData.html#PersistentDataStructure.newSubData"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.persistentData.PersistentDataStructure.newSubData" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>if key is not in database
|
|
create a new database (sqlitedict)
|
|
which can be queried from this one
|
|
via the key specified</p>
|
|
<p>this will automatically create a new
|
|
file where the filename is internally
|
|
managed (simple increasing number)</p>
|
|
</dd></dl>
|
|
|
|
<dl class="method">
|
|
<dt id="jobmanager.persistentData.PersistentDataStructure.open">
|
|
<tt class="descname">open</tt><big>(</big><big>)</big><a class="reference internal" href="_modules/jobmanager/persistentData.html#PersistentDataStructure.open"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.persistentData.PersistentDataStructure.open" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>open the SQL database at self._filename = <path>/__<name>/<name>.db
|
|
as sqlitedict</p>
|
|
</dd></dl>
|
|
|
|
<dl class="method">
|
|
<dt id="jobmanager.persistentData.PersistentDataStructure.setData">
|
|
<tt class="descname">setData</tt><big>(</big><em>key</em>, <em>value</em>, <em>overwrite=False</em><big>)</big><a class="reference internal" href="_modules/jobmanager/persistentData.html#PersistentDataStructure.setData"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.persistentData.PersistentDataStructure.setData" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>write the key value pair to the data base</p>
|
|
<p>if the key already exists, overwrite must be
|
|
set True in oder to update the data for
|
|
that key in the database</p>
|
|
</dd></dl>
|
|
|
|
<dl class="method">
|
|
<dt id="jobmanager.persistentData.PersistentDataStructure.setDataFromSubData">
|
|
<tt class="descname">setDataFromSubData</tt><big>(</big><em>key</em>, <em>subData</em><big>)</big><a class="reference internal" href="_modules/jobmanager/persistentData.html#PersistentDataStructure.setDataFromSubData"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#jobmanager.persistentData.PersistentDataStructure.setDataFromSubData" title="Permalink to this definition">¶</a></dt>
|
|
<dd><p>set an entry of the PDS with data from an other PDS</p>
|
|
<p>this means copying the appropriate file to the right place
|
|
and rename them</p>
|
|
</dd></dl>
|
|
|
|
</dd></dl>
|
|
|
|
</div>
|
|
</div>
|
|
<div class="section" id="examples">
|
|
<h2>Examples<a class="headerlink" href="#examples" title="Permalink to this headline">¶</a></h2>
|
|
<div class="section" id="module-simple_example">
|
|
<span id="simple-example"></span><h3>simple example<a class="headerlink" href="#module-simple_example" title="Permalink to this headline">¶</a></h3>
|
|
<p>Download: <a class="reference download internal" href="_downloads/simple_example.py"><tt class="xref download docutils literal"><span class="pre">../examples/simple_example.py</span></tt></a>.</p>
|
|
<div class="highlight-python"><div class="highlight"><pre><span class="c">#!/usr/bin/env python</span>
|
|
<span class="c"># -*- coding: utf-8 -*-</span>
|
|
<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">division</span><span class="p">,</span> <span class="n">print_function</span>
|
|
|
|
<span class="kn">import</span> <span class="nn">multiprocessing</span> <span class="kn">as</span> <span class="nn">mp</span>
|
|
<span class="kn">import</span> <span class="nn">numpy</span> <span class="kn">as</span> <span class="nn">np</span>
|
|
<span class="kn">from</span> <span class="nn">os.path</span> <span class="kn">import</span> <span class="n">split</span><span class="p">,</span> <span class="n">dirname</span><span class="p">,</span> <span class="n">abspath</span>
|
|
<span class="kn">import</span> <span class="nn">sys</span>
|
|
<span class="kn">import</span> <span class="nn">time</span>
|
|
|
|
<span class="c"># Add parent directory to beginning of path variable</span>
|
|
<span class="n">sys</span><span class="o">.</span><span class="n">path</span> <span class="o">=</span> <span class="p">[</span><span class="n">split</span><span class="p">(</span><span class="n">dirname</span><span class="p">(</span><span class="n">abspath</span><span class="p">(</span><span class="n">__file__</span><span class="p">)))[</span><span class="mi">0</span><span class="p">]]</span> <span class="o">+</span> <span class="n">sys</span><span class="o">.</span><span class="n">path</span>
|
|
|
|
<span class="kn">import</span> <span class="nn">jobmanager</span> <span class="kn">as</span> <span class="nn">jm</span>
|
|
|
|
|
|
|
|
<span class="k">class</span> <span class="nc">Example_Client</span><span class="p">(</span><span class="n">jm</span><span class="o">.</span><span class="n">JobManager_Client</span><span class="p">):</span>
|
|
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="c"># start quiet client (verbopse=0)</span>
|
|
<span class="nb">super</span><span class="p">(</span><span class="n">Example_Client</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">server</span><span class="o">=</span><span class="s">"localhost"</span><span class="p">,</span>
|
|
<span class="n">authkey</span><span class="o">=</span><span class="s">'simple example'</span><span class="p">,</span>
|
|
<span class="n">verbose</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
|
|
|
<span class="nd">@staticmethod</span>
|
|
<span class="k">def</span> <span class="nf">func</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="n">const_args</span><span class="p">):</span>
|
|
<span class="sd">"""simply return the current argument"""</span>
|
|
<span class="k">return</span> <span class="n">args</span>
|
|
|
|
|
|
|
|
<span class="k">class</span> <span class="nc">Example_Server</span><span class="p">(</span><span class="n">jm</span><span class="o">.</span><span class="n">JobManager_Server</span><span class="p">):</span>
|
|
<span class="k">def</span> <span class="nf">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="c"># server show status information (verbose=1)</span>
|
|
<span class="nb">super</span><span class="p">(</span><span class="n">Example_Server</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__init__</span><span class="p">(</span><span class="n">authkey</span><span class="o">=</span><span class="s">'simple example'</span><span class="p">,</span>
|
|
<span class="n">verbose</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
|
|
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">final_result</span> <span class="o">=</span> <span class="mi">1</span>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">process_new_result</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">arg</span><span class="p">,</span> <span class="n">result</span><span class="p">):</span>
|
|
<span class="sd">"""over write final_result with the new incoming result </span>
|
|
<span class="sd"> if the new result is smaller then the final_result"""</span>
|
|
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">final_result</span> <span class="o">></span> <span class="n">result</span><span class="p">:</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">final_result</span> <span class="o">=</span> <span class="n">result</span>
|
|
|
|
<span class="k">def</span> <span class="nf">process_final_result</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="k">print</span><span class="p">(</span><span class="s">"final_result:"</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">final_result</span><span class="p">)</span>
|
|
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">run_server</span><span class="p">():</span>
|
|
<span class="k">with</span> <span class="n">Example_Server</span><span class="p">()</span> <span class="k">as</span> <span class="n">server</span><span class="p">:</span>
|
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">5000</span><span class="p">):</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">put_arg</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">())</span>
|
|
<span class="n">server</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">run_client</span><span class="p">():</span>
|
|
<span class="n">client</span> <span class="o">=</span> <span class="n">Example_Client</span><span class="p">()</span>
|
|
<span class="n">client</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
|
|
|
|
|
|
<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s">"__main__"</span><span class="p">:</span>
|
|
<span class="n">p_server</span> <span class="o">=</span> <span class="n">mp</span><span class="o">.</span><span class="n">Process</span><span class="p">(</span><span class="n">target</span><span class="o">=</span><span class="n">run_server</span><span class="p">)</span>
|
|
<span class="n">p_server</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
|
|
|
|
<span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
|
|
|
<span class="n">p_client</span> <span class="o">=</span> <span class="n">mp</span><span class="o">.</span><span class="n">Process</span><span class="p">(</span><span class="n">target</span><span class="o">=</span><span class="n">run_client</span><span class="p">)</span>
|
|
<span class="n">p_client</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
|
|
|
|
<span class="n">p_client</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
|
|
<span class="n">p_server</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
|
|
</pre></div>
|
|
</div>
|
|
</div>
|
|
<div class="section" id="id10">
|
|
<h3>decorators<a class="headerlink" href="#id10" title="Permalink to this headline">¶</a></h3>
|
|
<span class="target" id="module-wrapper_example"></span><p>Download: <a class="reference download internal" href="_downloads/wrapper_example.py"><tt class="xref download docutils literal"><span class="pre">../examples/wrapper_example.py</span></tt></a>.</p>
|
|
<div class="highlight-python"><div class="highlight"><pre><span class="c">#!/usr/bin/env python</span>
|
|
<span class="c"># -*- coding: utf-8 -*-</span>
|
|
<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">division</span><span class="p">,</span> <span class="n">print_function</span>
|
|
<span class="sd">""" How to wrap or decorate a function with a progress bar.</span>
|
|
|
|
|
|
<span class="sd">"""</span>
|
|
|
|
|
|
<span class="kn">from</span> <span class="nn">os.path</span> <span class="kn">import</span> <span class="n">split</span><span class="p">,</span> <span class="n">dirname</span><span class="p">,</span> <span class="n">abspath</span>
|
|
<span class="kn">import</span> <span class="nn">sys</span>
|
|
<span class="kn">import</span> <span class="nn">time</span>
|
|
|
|
<span class="c"># Add parent directory to beginning of path variable</span>
|
|
<span class="n">sys</span><span class="o">.</span><span class="n">path</span> <span class="o">=</span> <span class="p">[</span><span class="n">split</span><span class="p">(</span><span class="n">dirname</span><span class="p">(</span><span class="n">abspath</span><span class="p">(</span><span class="n">__file__</span><span class="p">)))[</span><span class="mi">0</span><span class="p">]]</span> <span class="o">+</span> <span class="n">sys</span><span class="o">.</span><span class="n">path</span>
|
|
|
|
<span class="kn">import</span> <span class="nn">jobmanager</span> <span class="kn">as</span> <span class="nn">jm</span>
|
|
|
|
|
|
<span class="nd">@jm.decorators.ProgressBar</span>
|
|
<span class="k">def</span> <span class="nf">decorated_function_alpha</span><span class="p">(</span><span class="n">an_argument</span><span class="p">,</span> <span class="n">c</span><span class="p">,</span> <span class="n">m</span><span class="p">):</span>
|
|
<span class="sd">""" A simple example of a ProgressBar-decorated function.</span>
|
|
<span class="sd"> </span>
|
|
<span class="sd"> The arguments `c` and `m` are the counter and maximal counter</span>
|
|
<span class="sd"> variables of the ProgressBar. They are instances of</span>
|
|
<span class="sd"> `multiprocessing.Value`. </span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">m</span><span class="o">.</span><span class="n">value</span><span class="o">=</span><span class="mi">10</span>
|
|
<span class="n">c</span><span class="o">.</span><span class="n">value</span><span class="o">=</span><span class="mi">0</span>
|
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">):</span>
|
|
<span class="c"># this is were things are computed</span>
|
|
<span class="n">c</span><span class="o">.</span><span class="n">value</span> <span class="o">+=</span> <span class="mi">1</span>
|
|
<span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="o">.</span><span class="mi">2</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">an_argument</span>
|
|
|
|
|
|
<span class="nd">@jm.decorators.ProgressBar</span>
|
|
<span class="k">def</span> <span class="nf">decorated_function_beta</span><span class="p">(</span><span class="n">an_argument</span><span class="p">,</span> <span class="n">jmc</span><span class="p">,</span> <span class="n">jmm</span><span class="p">):</span>
|
|
<span class="sd">""" A simple example of a ProgressBar-decorated function.</span>
|
|
<span class="sd"> </span>
|
|
<span class="sd"> In comparison to `decorated_function_alpha`, we now have the</span>
|
|
<span class="sd"> arguments `jmc` and `jmm`. Jobmanager automatically detects</span>
|
|
<span class="sd"> arguments that are registered in</span>
|
|
<span class="sd"> `jobmanager.jobmanager.validCountKwargs`.</span>
|
|
|
|
<span class="sd"> Note that we do not need to set the value of jmc to zero, as the </span>
|
|
<span class="sd"> ProgressBar initiates the variable with zero.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">jmm</span><span class="o">.</span><span class="n">value</span><span class="o">=</span><span class="mi">10</span>
|
|
|
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">):</span>
|
|
<span class="c"># this is were things are computed</span>
|
|
<span class="n">jmc</span><span class="o">.</span><span class="n">value</span> <span class="o">+=</span> <span class="mi">1</span>
|
|
<span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="o">.</span><span class="mi">2</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">an_argument</span>
|
|
|
|
|
|
<span class="nd">@jm.decorators.ProgressBar</span>
|
|
<span class="k">def</span> <span class="nf">decorated_function_gamma</span><span class="p">(</span><span class="n">arg</span><span class="p">,</span> <span class="n">jmc</span><span class="p">,</span> <span class="n">jmm</span><span class="p">,</span> <span class="n">kwarg</span><span class="o">=</span><span class="s">"2"</span><span class="p">):</span>
|
|
<span class="sd">""" A simple example of a ProgressBar-decorated function.</span>
|
|
<span class="sd"> </span>
|
|
<span class="sd"> In comparison to `decorated_function_alpha`, we now have the</span>
|
|
<span class="sd"> arguments `jmc` and `jmm`. Jobmanager automatically detects</span>
|
|
<span class="sd"> arguments that are registered in</span>
|
|
<span class="sd"> `jobmanager.jobmanager.validCountKwargs`.</span>
|
|
|
|
<span class="sd"> Note that we do not need to set the value of jmc to zero, as the </span>
|
|
<span class="sd"> ProgressBar initiates the variable with zero.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="n">jmm</span><span class="o">.</span><span class="n">value</span><span class="o">=</span><span class="mi">10</span>
|
|
|
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">):</span>
|
|
<span class="c"># this is were things are computed</span>
|
|
<span class="n">jmc</span><span class="o">.</span><span class="n">value</span> <span class="o">+=</span> <span class="mi">1</span>
|
|
<span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="o">.</span><span class="mi">2</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="s">"{} {}"</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">arg</span><span class="p">,</span> <span class="n">kwarg</span><span class="p">)</span>
|
|
|
|
|
|
<span class="k">def</span> <span class="nf">wrapped_function_beta</span><span class="p">(</span><span class="n">an_argument</span><span class="p">,</span> <span class="n">jmc</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span> <span class="n">jmm</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
|
|
<span class="sd">""" A simple example of a ProgressBar-decorated function.</span>
|
|
<span class="sd"> </span>
|
|
<span class="sd"> In comparison to `decorated_function_beta`, the count arguments</span>
|
|
<span class="sd"> became keyword arguments. The function works with and without</span>
|
|
<span class="sd"> the ProgressBar.</span>
|
|
<span class="sd"> """</span>
|
|
<span class="k">if</span> <span class="n">jmm</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
|
|
<span class="n">jmm</span><span class="o">.</span><span class="n">value</span><span class="o">=</span><span class="mi">10</span>
|
|
|
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">):</span>
|
|
<span class="c"># this is were things are computed</span>
|
|
<span class="k">if</span> <span class="n">jmc</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">None</span><span class="p">:</span>
|
|
<span class="n">jmc</span><span class="o">.</span><span class="n">value</span> <span class="o">+=</span> <span class="mi">1</span>
|
|
<span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="o">.</span><span class="mi">2</span><span class="p">)</span>
|
|
<span class="k">return</span> <span class="n">an_argument</span>
|
|
|
|
|
|
|
|
<span class="k">if</span> <span class="n">__name__</span> <span class="o">==</span> <span class="s">"__main__"</span><span class="p">:</span>
|
|
<span class="c">##d ecorated</span>
|
|
<span class="n">retd1</span> <span class="o">=</span> <span class="n">decorated_function_alpha</span><span class="p">(</span><span class="s">"argument"</span><span class="p">)</span>
|
|
<span class="n">retd2</span> <span class="o">=</span> <span class="n">decorated_function_beta</span><span class="p">(</span><span class="s">"argument"</span><span class="p">)</span>
|
|
<span class="n">retd3</span> <span class="o">=</span> <span class="n">decorated_function_gamma</span><span class="p">(</span><span class="s">"argument"</span><span class="p">,</span> <span class="n">kwarg</span><span class="o">=</span><span class="s">"test"</span><span class="p">)</span>
|
|
<span class="c">## wrapped</span>
|
|
<span class="c"># When using the wrapper, you can define arguments for</span>
|
|
<span class="c"># `jm.progress.ProgressBar`.</span>
|
|
<span class="n">pb</span> <span class="o">=</span> <span class="n">jm</span><span class="o">.</span><span class="n">decorators</span><span class="o">.</span><span class="n">ProgressBar</span><span class="p">(</span><span class="n">wrapped_function_beta</span><span class="p">,</span>
|
|
<span class="n">interval</span><span class="o">=.</span><span class="mo">05</span><span class="p">)</span>
|
|
<span class="n">retw1</span> <span class="o">=</span> <span class="n">pb</span><span class="p">(</span><span class="s">"argument"</span><span class="p">)</span>
|
|
<span class="c"># or</span>
|
|
<span class="n">retw2</span> <span class="o">=</span> <span class="n">jm</span><span class="o">.</span><span class="n">decorators</span><span class="o">.</span><span class="n">ProgressBar</span><span class="p">(</span><span class="n">wrapped_function_beta</span><span class="p">)(</span><span class="s">"arg"</span><span class="p">)</span>
|
|
|
|
<span class="k">print</span><span class="p">(</span><span class="n">retd1</span><span class="p">,</span> <span class="n">retd2</span><span class="p">,</span> <span class="n">retd3</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s">" | "</span><span class="p">)</span>
|
|
<span class="k">print</span><span class="p">(</span><span class="n">retw1</span><span class="p">,</span> <span class="n">retw2</span><span class="p">,</span> <span class="n">sep</span><span class="o">=</span><span class="s">" | "</span><span class="p">)</span>
|
|
|
|
</pre></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="sphinxsidebar">
|
|
<div class="sphinxsidebarwrapper">
|
|
<h3><a href="#">Table Of Contents</a></h3>
|
|
<ul>
|
|
<li><a class="reference internal" href="#">jobmanager documentation</a><ul>
|
|
<li><a class="reference internal" href="#content">Content</a></li>
|
|
<li><a class="reference internal" href="#module-jobmanager">Main module</a></li>
|
|
<li><a class="reference internal" href="#module-jobmanager.progress">Progress monitoring</a><ul>
|
|
</ul>
|
|
</li>
|
|
<li><a class="reference internal" href="#user-convenience">User Convenience</a><ul>
|
|
<li><a class="reference internal" href="#module-jobmanager.decorators">Decorators</a><ul>
|
|
</ul>
|
|
</li>
|
|
<li><a class="reference internal" href="#module-jobmanager.ode_wrapper">ODE wrapper</a></li>
|
|
</ul>
|
|
</li>
|
|
<li><a class="reference internal" href="#core-modules">Core modules</a><ul>
|
|
<li><a class="reference internal" href="#jobmanager">jobmanager</a></li>
|
|
<li><a class="reference internal" href="#module-jobmanager.jobmanager">clients</a><ul>
|
|
</ul>
|
|
</li>
|
|
<li><a class="reference internal" href="#id7">clients</a><ul>
|
|
</ul>
|
|
</li>
|
|
<li><a class="reference internal" href="#module-jobmanager.servers">servers</a></li>
|
|
<li><a class="reference internal" href="#module-jobmanager.persistentData">persistentData</a></li>
|
|
</ul>
|
|
</li>
|
|
<li><a class="reference internal" href="#examples">Examples</a><ul>
|
|
<li><a class="reference internal" href="#module-simple_example">simple example</a></li>
|
|
<li><a class="reference internal" href="#id10">decorators</a></li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
</ul>
|
|
|
|
<h4>Next topic</h4>
|
|
<p class="topless"><a href=""
|
|
title="next chapter">jobmanager documentation</a></p>
|
|
<h3>This Page</h3>
|
|
<ul class="this-page-menu">
|
|
<li><a href="_sources/index.txt"
|
|
rel="nofollow">Show Source</a></li>
|
|
</ul>
|
|
<div id="searchbox" style="display: none">
|
|
<h3>Quick search</h3>
|
|
<form class="search" action="search.html" method="get">
|
|
<input type="text" name="q" />
|
|
<input type="submit" value="Go" />
|
|
<input type="hidden" name="check_keywords" value="yes" />
|
|
<input type="hidden" name="area" value="default" />
|
|
</form>
|
|
<p class="searchtip" style="font-size: 90%">
|
|
Enter search terms or a module, class or function name.
|
|
</p>
|
|
</div>
|
|
<script type="text/javascript">$('#searchbox').show(0);</script>
|
|
</div>
|
|
</div>
|
|
<div class="clearer"></div>
|
|
</div>
|
|
<div class="related">
|
|
<h3>Navigation</h3>
|
|
<ul>
|
|
<li class="right" style="margin-right: 10px">
|
|
<a href="genindex.html" title="General Index"
|
|
>index</a></li>
|
|
<li class="right" >
|
|
<a href="py-modindex.html" title="Python Module Index"
|
|
>modules</a> |</li>
|
|
<li class="right" >
|
|
<a href="#" title="jobmanager documentation"
|
|
>next</a> |</li>
|
|
<li><a href="#">jobmanager 0.1.0 documentation</a> »</li>
|
|
</ul>
|
|
</div>
|
|
<div class="footer">
|
|
© Copyright 2015, Richard Hartmann.
|
|
Created using <a href="http://sphinx-doc.org/">Sphinx</a> 1.2.2.
|
|
</div>
|
|
</body>
|
|
</html> |