Merge branch 'master' into new_decorator

Conflicts:
	jobmanager/progress.py

.gitignore

@@ -1,2 +1,11 @@
 __pycache__
 *.trb
+*.dump
+*.egg
+dist
+build
+*.py~
+*.md~
+*.pyc
+*.egg-info
+tests/pytest.html

README.md

@@ -3,14 +3,20 @@ jobmanager
 
 Easy distributed computing based on the python class SyncManager for remote communication and python module multiprocessing for local parallelism.
 
-### Testing on Linux
+### Developer's note
 After cloning into jobmanager, create a virtual environment
 
     virtualenv --system-site-packages ve_jm
     source ve_jm/bin/activate
 
-Install jobmanager into the environment and run a simple example.   
+Install all dependencies
+
+    python setup.py develop
+    
+Running an example
 
-    python setupy.py install
     python examples/simple_example.py
    
+Running tests
+
+    python setup.py test

jobmanager/jobmanager.py

@@ -844,7 +844,8 @@ class JobManager_Client(object):
             self.nproc = nproc
         else:
             self.nproc = mp.cpu_count() + nproc
-            assert self.nproc > 0
+            if self.nproc <= 0:
+                raise RuntimeError("Invalid Number of Processes\ncan not spawn {} processes (cores found: {}, cores NOT to use: {} = -nproc)".format(self.nproc, mp.cpu_count(), abs(nproc)))
 
         self.procs = []
         

run_pytest_html.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-OUTFILE='pytest.html'
-
-echo "run py.test ..."
-(date; py.test --color=yes) | aha --black --title "pytest output for jobmanager module"> $OUTFILE
-echo "done! (output written to $OUTFILE)"

setup.py

@@ -1,13 +1,24 @@
 #!/usr/bin/env python
 # To create a distribution package for pip or easy-install:
 # python setup.py sdist
-from setuptools import setup, find_packages
+from setuptools import setup, find_packages, Command
 from os.path import join, dirname, realpath
 from warnings import warn
 
 name='jobmanager'
 
+class PyTest(Command):
+    user_options = []
+    def initialize_options(self):
+        pass
 
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        import sys,subprocess
+        errno = subprocess.call([sys.executable, 'tests/runtests.py'])
+        raise SystemExit(errno)
 
 
 
@@ -38,16 +49,18 @@ and python module multiprocessing for local
 parallelism.""",
     install_requires=["sqlitedict", "NumPy>=1.5.1"],
     # tests: psutil
-    keywords=["multiprocessing", "queue", "parallel",
-              "progress", "manager", "job"],
+    keywords=["multiprocessing", "queue", "parallel", "distributed", "computing",
+              "progress", "manager", "job", "persistent data"],
     classifiers= [
         'Operating System :: OS Independent',
         #'Programming Language :: Python :: 2.7', #Todo
-        'Programming Language :: Python :: 3.2',
-        'Programming Language :: Python :: 3.3',
+        #'Programming Language :: Python :: 3.2', # also not very well tested
+        #'Programming Language :: Python :: 3.3', # also not very well tested
+        'Programming Language :: Python :: 3.4',
         'Intended Audience :: Science/Research'
                  ],
-    platforms=['ALL']
+    platforms=['ALL'],
+    cmdclass = {'test': PyTest},
     )
 
 

test/README.md

@@ -1,29 +0,0 @@
-Test Scripts
-============
-
-Execute the scripts using python or use pytest.
-
-Client
-------
-
-    python test_clients.py
-
-
-Progress
---------
-
-    python test_progress.py
-
-
-
-Jobmanager Server-Client
-------------------------
-
-Server:
-
-    python test_jobmanager.py server
-    
-
-Client
-
-    python test_jobmanager.py

tests/README.md

@@ -0,0 +1,41 @@
+### Test Scripts
+
+
+This will run all tests:
+
+    python runtests.py
+
+Beautiful html output is possible with (Unix, package `aha` required)
+
+    ./runtests_html.sh
+
+
+### Running single tests
+
+Directly execute the scripts, e.g.
+
+
+#### Client
+
+
+    python test_clients.py
+
+
+#### Progress
+
+
+    python test_progress.py
+
+
+
+#### Jobmanager Server-Client
+
+
+Server:
+
+    python test_jobmanager.py server
+    
+
+Client
+
+    python test_jobmanager.py

tests/runtests_html.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+# Go to script directory
+cd $(dirname $0)
+
+OUTFILE='pytest.html'
+
+echo "Working directory: $(pwd)"
+echo "Running py.test ..."
+(date; python runtests.py --color=yes) | aha --black --title "pytest output for jobmanager module"> $OUTFILE
+echo "Done! (output written to $OUTFILE)"

tests/test_progress.py

@@ -479,7 +479,7 @@ if __name__ == "__main__":
 #     test_progress_bar_counter,
 #     test_progress_bar_counter_non_max,
 #     test_progress_bar_counter_hide_bar,
-    test_progress_bar_slow_change,
+#     test_progress_bar_slow_change,
     lambda: print("END")
     ]
     

tests/test_servers.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import sys
+from os.path import abspath, dirname, split
+
+# Add parent directory to beginning of path variable
+sys.path = [split(dirname(abspath(__file__)))[0]] + sys.path
+
+import jobmanager as jm
+
+def bin_list():
+    import pickle
+    a =  ['cvbnm', 'qwert', 'asdfg']
+    print(pickle.dumps(a))
+    
+def bin_dict():
+    import pickle
+    a =  {'cvbnm': 1, 'qwert': 2, 'asdfg': 3}
+    print(pickle.dumps(a))
+    
+def see_bin_data():
+    import multiprocessing as mp
+    mp.set_start_method('spawn')
+    
+    for i in range(10):
+        p = mp.Process(target = bin_list)
+        p.start()
+        p.join()
+        
+    for i in range(10):
+        p = mp.Process(target = bin_dict)
+        p.start()
+        p.join()
+
+def test_recursive_type_scan():
+    d = {'a': 1, 'b':2}
+    l = [2, 3]
+    t = ('3','4')
+    i = 1
+    f = 3.4
+    s = 's'
+    
+    a1 = [l, t, d]
+    a2 = [l, t]
+    
+    a3 = (d, d, d)
+    a4 = (i, l, a1)
+    
+    
+    ###
+    # GENERAL DICT SCAN
+    ###
+    assert jm.servers.recursive_scan_for_instance(obj=d, type=dict) == True
+    
+    assert jm.servers.recursive_scan_for_instance(obj=l, type=dict) == False
+    assert jm.servers.recursive_scan_for_instance(obj=t, type=dict) == False
+    assert jm.servers.recursive_scan_for_instance(obj=i, type=dict) == False
+    assert jm.servers.recursive_scan_for_instance(obj=f, type=dict) == False
+    assert jm.servers.recursive_scan_for_instance(obj=s, type=dict) == False
+    
+    assert jm.servers.recursive_scan_for_instance(obj=a1, type=dict) == True
+    assert jm.servers.recursive_scan_for_instance(obj=a2, type=dict) == False
+    assert jm.servers.recursive_scan_for_instance(obj=a3, type=dict) == True
+    assert jm.servers.recursive_scan_for_instance(obj=a4, type=dict) == True
+    
+    ###
+    # SPECIFIC DICT SCAN
+    ###
+    assert jm.servers.recursive_scan_for_dict_instance(obj=d) == True
+    
+    assert jm.servers.recursive_scan_for_dict_instance(obj=l) == False
+    assert jm.servers.recursive_scan_for_dict_instance(obj=t) == False
+    assert jm.servers.recursive_scan_for_dict_instance(obj=i) == False
+    assert jm.servers.recursive_scan_for_dict_instance(obj=f) == False
+    assert jm.servers.recursive_scan_for_dict_instance(obj=s) == False
+    
+    assert jm.servers.recursive_scan_for_dict_instance(obj=a1) == True
+    assert jm.servers.recursive_scan_for_dict_instance(obj=a2) == False
+    assert jm.servers.recursive_scan_for_dict_instance(obj=a3) == True
+    assert jm.servers.recursive_scan_for_dict_instance(obj=a4) == True
+
+    ###
+    # INT SCAN
+    ###
+    
+    assert jm.servers.recursive_scan_for_instance(obj = i, type=int) == True
+    assert jm.servers.recursive_scan_for_instance(obj = l, type=int) == True
+    assert jm.servers.recursive_scan_for_instance(obj = a1, type=int) == True
+    assert jm.servers.recursive_scan_for_instance(obj = a2, type=int) == True
+    assert jm.servers.recursive_scan_for_instance(obj = a4, type=int) == True
+    
+    print(jm.servers.recursive_scan_for_instance(obj = d, type=int))
+    print(jm.servers.recursive_scan_for_instance(obj = a3, type=int))
+    
+    for i in d:
+        print(i)
+    
+    
+if __name__ == "__main__":
+    test_recursive_type_scan()
+#     see_bin_data()