mirror of
https://github.com/vale981/jobmanager
synced 2025-03-05 09:51:38 -05:00
Merge pull request #25 from paulmueller/master
Travis automated testing
This commit is contained in:
Paul Müller
commit
3033572c16
5 changed files with 76 additions and 17 deletions
22
.travis.yml
Normal file
22
.travis.yml
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
language: python
|
||||
notifications:
|
||||
email: false
|
||||
python:
|
||||
- "3.4"
|
||||
before_install:
|
||||
- sudo apt-get update -qq
|
||||
- sudo apt-get install -qq libatlas-dev libatlas-base-dev liblapack-dev gfortran
|
||||
- wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh
|
||||
- chmod +x miniconda.sh
|
||||
- ./miniconda.sh -b
|
||||
- export PATH=/home/travis/miniconda/bin:$PATH
|
||||
- conda update --yes conda
|
||||
- travis_retry conda install --yes python=$TRAVIS_PYTHON_VERSION pip numpy scipy matplotlib
|
||||
install:
|
||||
- travis_retry pip install sqlitedict
|
||||
- travis_retry pip install psutil
|
||||
- pip freeze
|
||||
script:
|
||||
- netstat -tulpen
|
||||
- travis_wait python setup.py test
|
||||
|
||||
|
|
@ -1,5 +1,8 @@
|
|||
jobmanager
|
||||
==========
|
||||
[](https://pypi.python.org/pypi/jobmanager)
|
||||
[](https://travis-ci.org/cimatosa/jobmanager)
|
||||
|
||||
|
||||
Easy distributed computing based on the python class SyncManager for remote communication and python module multiprocessing for local parallelism.
|
||||
|
||||
|
|
|
|||
2
setup.py
2
setup.py
|
|
@ -70,7 +70,7 @@ if __name__ == "__main__":
|
|||
description=description,
|
||||
long_description=longdescription,
|
||||
install_requires=["sqlitedict>=1.2.0", "NumPy>=1.5.1"],
|
||||
tests_require=["psutil"],
|
||||
tests_require=["psutil", "scipy"],
|
||||
keywords=["multiprocessing", "queue", "parallel", "distributed", "computing",
|
||||
"progress", "manager", "job", "persistent data", "scheduler"],
|
||||
classifiers= [
|
||||
|
|
|
|||
|
|
@ -126,7 +126,7 @@ def test_distributed_mathieu():
|
|||
|
||||
with jm.JobManager_Local(client_class = jm.clients.Integration_Client_REAL,
|
||||
authkey = authkey,
|
||||
port = 42525,
|
||||
port = 42520,
|
||||
const_arg = const_arg,
|
||||
nproc=1,
|
||||
verbose_client=2,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import signal
|
|||
import multiprocessing as mp
|
||||
import numpy as np
|
||||
import traceback
|
||||
import socket
|
||||
import subprocess
|
||||
import signal
|
||||
|
||||
|
|
@ -18,11 +19,14 @@ sys.path = [split(dirname(abspath(__file__)))[0]] + sys.path
|
|||
|
||||
from jobmanager import jobmanager, progress
|
||||
|
||||
PORT = 42525
|
||||
AUTHKEY = 'testing'
|
||||
PORT = 42525
|
||||
SERVER = socket.gethostname()
|
||||
|
||||
|
||||
def test_Signal_to_SIG_IGN():
|
||||
global PORT
|
||||
PORT += 1
|
||||
def f():
|
||||
jobmanager.Signal_to_SIG_IGN()
|
||||
print("before sleep")
|
||||
|
|
@ -56,6 +60,8 @@ def test_Signal_to_SIG_IGN():
|
|||
print("[+] terminated")
|
||||
|
||||
def test_Signal_to_sys_exit():
|
||||
global PORT
|
||||
PORT += 1
|
||||
def f():
|
||||
jobmanager.Signal_to_sys_exit()
|
||||
while True:
|
||||
|
|
@ -91,6 +97,8 @@ def test_Signal_to_sys_exit():
|
|||
print("[+] terminated")
|
||||
|
||||
def test_Signal_to_terminate_process_list():
|
||||
global PORT
|
||||
PORT += 1
|
||||
def child_proc():
|
||||
jobmanager.Signal_to_sys_exit()
|
||||
try:
|
||||
|
|
@ -136,7 +144,7 @@ def start_server(n, read_old_state=False, verbose=1):
|
|||
|
||||
def start_client(verbose=1):
|
||||
print("START CLIENT")
|
||||
jm_client = jobmanager.JobManager_Client(server = 'localhost',
|
||||
jm_client = jobmanager.JobManager_Client(server = SERVER,
|
||||
authkey = AUTHKEY,
|
||||
port = PORT,
|
||||
nproc = 0,
|
||||
|
|
@ -151,6 +159,8 @@ def test_jobmanager_basic():
|
|||
|
||||
check if all arguments are found in final_result of dump
|
||||
"""
|
||||
global PORT
|
||||
PORT += 1
|
||||
n = 10
|
||||
p_server = mp.Process(target=start_server, args=(n,))
|
||||
p_server.start()
|
||||
|
|
@ -183,6 +193,8 @@ def test_jobmanager_basic():
|
|||
|
||||
|
||||
def test_jobmanager_server_signals():
|
||||
global PORT
|
||||
PORT += 1
|
||||
print("## TEST SIGTERM ##")
|
||||
p_server = mp.Process(target=start_server, args=(30,))
|
||||
p_server.start()
|
||||
|
|
@ -206,7 +218,7 @@ def test_jobmanager_server_signals():
|
|||
assert len(ref_set - args_set) == 0
|
||||
print("[+] args_set from dump contains all arguments")
|
||||
|
||||
|
||||
PORT += 1
|
||||
print("## TEST SIGINT ##")
|
||||
p_server = mp.Process(target=start_server, args=(30,))
|
||||
p_server.start()
|
||||
|
|
@ -243,6 +255,8 @@ def test_shutdown_server_while_client_running():
|
|||
check if the final_result and the args dump end up to include
|
||||
all arguments given
|
||||
"""
|
||||
global PORT
|
||||
PORT += 1
|
||||
|
||||
n = 1000
|
||||
|
||||
|
|
@ -251,6 +265,7 @@ def test_shutdown_server_while_client_running():
|
|||
|
||||
time.sleep(1)
|
||||
|
||||
PORT += 1
|
||||
p_client = mp.Process(target=start_client, args=(2,))
|
||||
p_client.start()
|
||||
|
||||
|
|
@ -313,7 +328,8 @@ def shutdown_client(sig):
|
|||
if server does not terminate on time, something must be wrong with args_set
|
||||
check if the final_result contain all arguments given
|
||||
"""
|
||||
|
||||
global PORT
|
||||
PORT += 1
|
||||
n = 300
|
||||
|
||||
print("## terminate client with {} ##".format(progress.signal_dict[sig]))
|
||||
|
|
@ -366,6 +382,8 @@ def shutdown_client(sig):
|
|||
print("[+] all arguments found in final_results")
|
||||
|
||||
def test_check_fail():
|
||||
global PORT
|
||||
PORT += 1
|
||||
class Client_Random_Error(jobmanager.JobManager_Client):
|
||||
def func(self, args, const_args, c, m):
|
||||
c.value = 0
|
||||
|
|
@ -385,7 +403,7 @@ def test_check_fail():
|
|||
time.sleep(1)
|
||||
|
||||
print("START CLIENT")
|
||||
jm_client = Client_Random_Error(server='localhost',
|
||||
jm_client = Client_Random_Error(server=SERVER,
|
||||
authkey=AUTHKEY,
|
||||
port=PORT,
|
||||
nproc=0,
|
||||
|
|
@ -438,6 +456,8 @@ def test_jobmanager_read_old_stat():
|
|||
|
||||
check if all arguments are found in final_result of dump
|
||||
"""
|
||||
global PORT
|
||||
PORT += 1
|
||||
n = 100
|
||||
p_server = mp.Process(target=start_server, args=(n,))
|
||||
p_server.start()
|
||||
|
|
@ -459,7 +479,7 @@ def test_jobmanager_read_old_stat():
|
|||
print("[+] client and server terminated")
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
PORT += 1
|
||||
p_server = mp.Process(target=start_server, args=(n,True))
|
||||
p_server.start()
|
||||
|
||||
|
|
@ -546,6 +566,8 @@ def test_hashedViewOnNumpyArray():
|
|||
assert bh2 in s
|
||||
|
||||
def test_client_status():
|
||||
global PORT
|
||||
PORT += 1
|
||||
n = 10
|
||||
p_server = mp.Process(target=start_server, args=(n,False,0))
|
||||
p_server.start()
|
||||
|
|
@ -561,7 +583,7 @@ def test_client_status():
|
|||
|
||||
return os.getpid()
|
||||
|
||||
client = Client_With_Status(server = 'localhost',
|
||||
client = Client_With_Status(server = SERVER,
|
||||
authkey = AUTHKEY,
|
||||
port = PORT,
|
||||
nproc = 4,
|
||||
|
|
@ -570,6 +592,8 @@ def test_client_status():
|
|||
p_server.join()
|
||||
|
||||
def test_jobmanager_local():
|
||||
global PORT
|
||||
PORT += 1
|
||||
args = range(1,200)
|
||||
with jobmanager.JobManager_Local(client_class = jobmanager.JobManager_Client,
|
||||
authkey = AUTHKEY,
|
||||
|
|
@ -581,6 +605,8 @@ def test_jobmanager_local():
|
|||
jm_server.start()
|
||||
|
||||
def test_start_server_on_used_port():
|
||||
global PORT
|
||||
PORT += 1
|
||||
def start_server():
|
||||
const_arg = None
|
||||
arg = [10,20,30]
|
||||
|
|
@ -623,6 +649,8 @@ def test_start_server_on_used_port():
|
|||
assert not other_error
|
||||
|
||||
def test_shared_const_arg():
|
||||
global PORT
|
||||
PORT += 1
|
||||
def start_server():
|
||||
const_arg = {1:1, 2:2, 3:3}
|
||||
arg = [10,20,30]
|
||||
|
|
@ -643,7 +671,7 @@ def test_shared_const_arg():
|
|||
print(os.getpid(), arg, const_arg)
|
||||
return None
|
||||
|
||||
client = myClient(server='localhost',
|
||||
client = myClient(server=SERVER,
|
||||
authkey=AUTHKEY,
|
||||
port = PORT,
|
||||
nproc=1,
|
||||
|
|
@ -651,6 +679,7 @@ def test_shared_const_arg():
|
|||
|
||||
client.start()
|
||||
|
||||
PORT += 1
|
||||
p1 = mp.Process(target=start_server)
|
||||
p2 = mp.Process(target=start_client)
|
||||
|
||||
|
|
@ -666,6 +695,8 @@ def test_shared_const_arg():
|
|||
p1.join()
|
||||
|
||||
def test_digest_rejected():
|
||||
global PORT
|
||||
PORT += 1
|
||||
n = 10
|
||||
p_server = mp.Process(target=start_server, args=(n,False,0))
|
||||
p_server.start()
|
||||
|
|
@ -681,7 +712,7 @@ def test_digest_rejected():
|
|||
|
||||
return os.getpid()
|
||||
|
||||
client = Client_With_Status(server = 'localhost',
|
||||
client = Client_With_Status(server = SERVER,
|
||||
authkey = AUTHKEY+' not the same',
|
||||
port = PORT,
|
||||
nproc = 4,
|
||||
|
|
@ -694,7 +725,9 @@ def test_digest_rejected():
|
|||
|
||||
p_server.join()
|
||||
|
||||
def test_exception():
|
||||
def test_exception():
|
||||
global PORT
|
||||
PORT += 1
|
||||
class MyManager_Client(jobmanager.BaseManager):
|
||||
pass
|
||||
|
||||
|
|
@ -729,7 +762,8 @@ def test_exception():
|
|||
return m
|
||||
|
||||
for p_version_server in [2, 3]:
|
||||
port = np.random.randint(20000, 30000)
|
||||
PORT += 10
|
||||
port = PORT
|
||||
authkey = 'q'
|
||||
with open("ap_server.out", 'w') as outfile:
|
||||
|
||||
|
|
@ -742,7 +776,7 @@ def test_exception():
|
|||
|
||||
try:
|
||||
try:
|
||||
autoproxy_connect(server='localhost', port=port, authkey=authkey)
|
||||
autoproxy_connect(server=SERVER, port=port, authkey=authkey)
|
||||
except jobmanager.RemoteValueError:
|
||||
if (sys.version_info[0] == 3) and (p_version_server == 2):
|
||||
print("that is ok") # the occurrence of this Exception is normal
|
||||
|
|
@ -761,20 +795,20 @@ def test_exception():
|
|||
continue
|
||||
|
||||
try:
|
||||
autoproxy_connect(server='localhost', port=port+1, authkey=authkey)
|
||||
autoproxy_connect(server=SERVER, port=port+1, authkey=authkey)
|
||||
except jobmanager.JMConnectionRefusedError:
|
||||
print("that is ok")
|
||||
except:
|
||||
raise
|
||||
|
||||
try:
|
||||
autoproxy_connect(server='localhost', port=port, authkey=authkey+'_')
|
||||
autoproxy_connect(server=SERVER, port=port, authkey=authkey+'_')
|
||||
except jobmanager.AuthenticationError:
|
||||
print("that is ok")
|
||||
except:
|
||||
raise
|
||||
|
||||
m = autoproxy_connect(server='localhost', port=port, authkey=authkey)
|
||||
m = autoproxy_connect(server=SERVER, port=port, authkey=authkey)
|
||||
|
||||
q = m.get_q()
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue