From 06bc6d4521a0ed37488ce8cc631f820efb592bdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Sun, 26 Apr 2015 15:49:36 +0200 Subject: [PATCH 01/56] travis initial commit --- .travis.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..26b128c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,28 @@ +# Mac OS X Python build +language: +- objective-c +python: +- "3.4" +# Only build master branch +branches: + only: + - master +install: +# install python via brew +- which python +- python --version +# pip init +- pip install --upgrade pip +# use anaconda +- pip install conda +- conda init +- conda install --yes 'numpy<1.9.0' +# final pip installs +- pip install wheel +# show packages +- pip freeze +# build extensions +- python setup.py develop +script: +# Create wheel pacakge +- python setup.py test From ffdf9b827bebdf1debdd517c3f25f3c1d4d63f5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Sun, 26 Apr 2015 15:50:18 +0200 Subject: [PATCH 02/56] travis on linux --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 26b128c..a0d9b37 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,3 @@ -# Mac OS X Python build -language: -- objective-c python: - "3.4" # Only build master branch From 5e557170f4156c1eaeb2e76d5ead7675fe1c26b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Sun, 26 Apr 2015 15:51:22 +0200 Subject: [PATCH 03/56] travis edit --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a0d9b37..2afe636 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,10 +16,10 @@ install: - conda install --yes 'numpy<1.9.0' # final pip installs - pip install wheel -# show packages -- pip freeze # build extensions - python setup.py develop +# show packages +- pip freeze script: # Create wheel pacakge - python setup.py test From 877cfe94c68663ec7670f07ede3ee67c2301a32a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Sun, 26 Apr 2015 15:53:25 +0200 Subject: [PATCH 04/56] travis sudo --- .travis.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2afe636..8423be8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,13 +9,13 @@ install: - which python - python --version # pip init -- pip install --upgrade pip +- sudo pip install --upgrade pip # use anaconda -- pip install conda -- conda init -- conda install --yes 'numpy<1.9.0' +- sudo pip install conda +- sudo conda init +- sudo conda install --yes 'numpy<1.9.0' # final pip installs -- pip install wheel +- sudo pip install wheel # build extensions - python setup.py develop # show packages From 96c02df82ed266699b7e251acfa33f93cf5a27e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Sun, 26 Apr 2015 15:55:20 +0200 Subject: [PATCH 05/56] travis update --- .travis.yml | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8423be8..e0cc021 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,25 +1,21 @@ +language: python python: - "3.4" # Only build master branch branches: only: - master +before_install: +- sudo apt-get update -qq +- wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh +- chmod +x miniconda.sh +- ./miniconda.sh -b +- export PATH=/home/travis/miniconda/bin:$PATH +- conda update --yes conda +- travis_retry conda install --yes python=$TRAVIS_PYTHON_VERSION pip numpy install: -# install python via brew -- which python -- python --version -# pip init -- sudo pip install --upgrade pip -# use anaconda -- sudo pip install conda -- sudo conda init -- sudo conda install --yes 'numpy<1.9.0' -# final pip installs -- sudo pip install wheel -# build extensions -- python setup.py develop -# show packages -- pip freeze +- travis_retry python setup.py develop script: # Create wheel pacakge -- python setup.py test +- travis_wait python setup.py test +cache: apt From f9fb4b9ca080d57c8a18c440d82e97c56bfa220d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Sun, 26 Apr 2015 16:01:07 +0200 Subject: [PATCH 06/56] shields --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 191eb4e..b8104e6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ jobmanager ========== +[![PyPI](http://img.shields.io/pypi/v/jobmanager.svg)](https://pypi.python.org/pypi/jobmanager) +[![Travis](http://img.shields.io/travis/cimatosa/jobmanager.svg?label=tests)](https://travis-ci.org/cimatosa/jobmanager) + Easy distributed computing based on the python class SyncManager for remote communication and python module multiprocessing for local parallelism. From 864a90398559216cbc633a21409cc754fffcbbb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Sun, 26 Apr 2015 16:05:40 +0200 Subject: [PATCH 07/56] travis verbose --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index e0cc021..2ef75b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,6 +13,8 @@ before_install: - export PATH=/home/travis/miniconda/bin:$PATH - conda update --yes conda - travis_retry conda install --yes python=$TRAVIS_PYTHON_VERSION pip numpy +- python --version +- pip freeze install: - travis_retry python setup.py develop script: From ecd917cf42edafee1bba4fae8028d093d7c96da5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Sun, 26 Apr 2015 16:10:11 +0200 Subject: [PATCH 08/56] travis miniconda no needed --- .travis.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2ef75b5..03e68b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,12 +7,8 @@ branches: - master before_install: - sudo apt-get update -qq -- wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh -- chmod +x miniconda.sh -- ./miniconda.sh -b -- export PATH=/home/travis/miniconda/bin:$PATH -- conda update --yes conda -- travis_retry conda install --yes python=$TRAVIS_PYTHON_VERSION pip numpy +- if [[ $TRAVIS_PYTHON_VERSION == "2.7" ]]; then sudo apt-get install python-numpy; fi +- if [[ $TRAVIS_PYTHON_VERSION == "3.4" ]]; then sudo apt-get install python3-numpy; fi - python --version - pip freeze install: From 7e16b579b69cab18ba1a6792a42b651dd2be7c67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Sun, 26 Apr 2015 16:11:19 +0200 Subject: [PATCH 09/56] travis virtualenv --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 03e68b8..334263d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,14 +5,16 @@ python: branches: only: - master +virtualenv: + system_site_packages: true before_install: - sudo apt-get update -qq +install: - if [[ $TRAVIS_PYTHON_VERSION == "2.7" ]]; then sudo apt-get install python-numpy; fi - if [[ $TRAVIS_PYTHON_VERSION == "3.4" ]]; then sudo apt-get install python3-numpy; fi +- travis_retry python setup.py develop - python --version - pip freeze -install: -- travis_retry python setup.py develop script: # Create wheel pacakge - travis_wait python setup.py test From b34ffd6b244f2bdafcade1d9ac3f091156abac56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Sun, 26 Apr 2015 16:20:52 +0200 Subject: [PATCH 10/56] travis: no system site packages --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 334263d..b2407bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,6 @@ python: branches: only: - master -virtualenv: - system_site_packages: true before_install: - sudo apt-get update -qq install: @@ -16,6 +14,5 @@ install: - python --version - pip freeze script: -# Create wheel pacakge - travis_wait python setup.py test cache: apt From 8831ac3d4fffa0beeb4507a68ed15ce168bf6e32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Sun, 26 Apr 2015 16:35:02 +0200 Subject: [PATCH 11/56] travis: need to use conda --- .travis.yml | 10 ++++++++-- setup.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b2407bb..44a00fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,10 +6,16 @@ branches: only: - master before_install: +# We need to use conda, because virtualenv did not work on travis with 3.4 - sudo apt-get update -qq +- sudo apt-get install -qq libatlas-dev libatlas-base-dev liblapack-dev gfortran +- wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh +- chmod +x miniconda.sh +- ./miniconda.sh -b +- export PATH=/home/travis/miniconda/bin:$PATH +- conda update --yes conda +- travis_retry conda install --yes python=$TRAVIS_PYTHON_VERSION pip numpy scipy install: -- if [[ $TRAVIS_PYTHON_VERSION == "2.7" ]]; then sudo apt-get install python-numpy; fi -- if [[ $TRAVIS_PYTHON_VERSION == "3.4" ]]; then sudo apt-get install python3-numpy; fi - travis_retry python setup.py develop - python --version - pip freeze diff --git a/setup.py b/setup.py index 0b89641..4232f4f 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ if __name__ == "__main__": description=description, long_description=longdescription, install_requires=["sqlitedict>=1.2.0", "NumPy>=1.5.1"], - tests_require=["psutil"], + tests_require=["psutil, scipy"], keywords=["multiprocessing", "queue", "parallel", "distributed", "computing", "progress", "manager", "job", "persistent data", "scheduler"], classifiers= [ From 1c9bda128a0aba0be9bc85315d9bca3b9a9a1b79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Sun, 26 Apr 2015 16:37:55 +0200 Subject: [PATCH 12/56] travis: typo in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4232f4f..4757204 100644 --- a/setup.py +++ b/setup.py @@ -70,7 +70,7 @@ if __name__ == "__main__": description=description, long_description=longdescription, install_requires=["sqlitedict>=1.2.0", "NumPy>=1.5.1"], - tests_require=["psutil, scipy"], + tests_require=["psutil", "scipy"], keywords=["multiprocessing", "queue", "parallel", "distributed", "computing", "progress", "manager", "job", "persistent data", "scheduler"], classifiers= [ From 9c43a60fd6a11fd973250ffe003b084c1e4377eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Sun, 26 Apr 2015 18:32:42 +0200 Subject: [PATCH 13/56] travis failing, try root --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 44a00fc..cd9df24 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,5 +20,5 @@ install: - python --version - pip freeze script: -- travis_wait python setup.py test +- travis_wait sudo python setup.py test cache: apt From 0f0e3a2830004f07cadd221590b83fe3e5dd0a76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Sun, 26 Apr 2015 18:36:04 +0200 Subject: [PATCH 14/56] travis: try all sudo --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cd9df24..53cbf0a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ before_install: # We need to use conda, because virtualenv did not work on travis with 3.4 - sudo apt-get update -qq - sudo apt-get install -qq libatlas-dev libatlas-base-dev liblapack-dev gfortran +- sudo -s - wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh - chmod +x miniconda.sh - ./miniconda.sh -b @@ -20,5 +21,5 @@ install: - python --version - pip freeze script: -- travis_wait sudo python setup.py test +- travis_wait python setup.py test cache: apt From 2f920de90996a6412feffb5d9f1978ffec0f244a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Sun, 26 Apr 2015 18:43:24 +0200 Subject: [PATCH 15/56] travis: sudo for each command --- .travis.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 53cbf0a..75118ec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,17 +9,16 @@ before_install: # We need to use conda, because virtualenv did not work on travis with 3.4 - sudo apt-get update -qq - sudo apt-get install -qq libatlas-dev libatlas-base-dev liblapack-dev gfortran -- sudo -s -- wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh -- chmod +x miniconda.sh -- ./miniconda.sh -b +- sudo wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh +- sudo chmod +x miniconda.sh +- sudo ./miniconda.sh -b - export PATH=/home/travis/miniconda/bin:$PATH -- conda update --yes conda -- travis_retry conda install --yes python=$TRAVIS_PYTHON_VERSION pip numpy scipy +- sudo conda update --yes conda +- travis_retry sudo conda install --yes python=$TRAVIS_PYTHON_VERSION pip numpy scipy install: -- travis_retry python setup.py develop +- travis_retry sudo python setup.py develop - python --version - pip freeze script: -- travis_wait python setup.py test +- travis_wait sudo python setup.py test cache: apt From 547448a9a81206d8ccc18d1ed4146b0c700822bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Sun, 26 Apr 2015 18:47:44 +0200 Subject: [PATCH 16/56] travis: manually select python version --- .travis.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 75118ec..334f1ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,16 +9,17 @@ before_install: # We need to use conda, because virtualenv did not work on travis with 3.4 - sudo apt-get update -qq - sudo apt-get install -qq libatlas-dev libatlas-base-dev liblapack-dev gfortran -- sudo wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh -- sudo chmod +x miniconda.sh -- sudo ./miniconda.sh -b +- wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh +- chmod +x miniconda.sh +- ./miniconda.sh -b - export PATH=/home/travis/miniconda/bin:$PATH -- sudo conda update --yes conda -- travis_retry sudo conda install --yes python=$TRAVIS_PYTHON_VERSION pip numpy scipy +- conda update --yes conda +- travis_retry conda install --yes python=$TRAVIS_PYTHON_VERSION pip numpy scipy install: -- travis_retry sudo python setup.py develop +- travis_retry python setup.py develop - python --version +- /home/travis/miniconda/bin/python --version - pip freeze script: -- travis_wait sudo python setup.py test +- travis_wait sudo /home/travis/miniconda/bin/python setup.py test cache: apt From 4a8169c283b7b396e338400ba35701d3b1f2fd5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Tue, 28 Apr 2015 10:40:26 +0200 Subject: [PATCH 17/56] travis trigger build --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 334f1ae..46de108 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: - "3.4" -# Only build master branch +# Only build master branch. branches: only: - master From 312b3ee219798db080623b6118aa0785ace045c4 Mon Sep 17 00:00:00 2001 From: paulmueller Date: Mon, 11 May 2015 23:53:23 +0200 Subject: [PATCH 18/56] travis with apt-get --- .travis.yml | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 46de108..a4a40f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,25 +1,12 @@ language: python python: - "3.4" -# Only build master branch. -branches: - only: - - master before_install: # We need to use conda, because virtualenv did not work on travis with 3.4 - sudo apt-get update -qq -- sudo apt-get install -qq libatlas-dev libatlas-base-dev liblapack-dev gfortran -- wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh -- chmod +x miniconda.sh -- ./miniconda.sh -b -- export PATH=/home/travis/miniconda/bin:$PATH -- conda update --yes conda -- travis_retry conda install --yes python=$TRAVIS_PYTHON_VERSION pip numpy scipy -install: -- travis_retry python setup.py develop -- python --version -- /home/travis/miniconda/bin/python --version +- if [[ $TRAVIS_PYTHON_VERSION == "2.7" ]]; then sudo apt-get -y install python-numpy; fi +- if [[ $TRAVIS_PYTHON_VERSION == "3.4" ]]; then sudo apt-get -y install python3-numpy; fi - pip freeze script: -- travis_wait sudo /home/travis/miniconda/bin/python setup.py test -cache: apt +- travis_wait python setup.py test + From e751d04c5ea5f42e936a6701f84c3a64c23de4f1 Mon Sep 17 00:00:00 2001 From: paulmueller Date: Tue, 12 May 2015 00:06:54 +0200 Subject: [PATCH 19/56] trying to test on travis ci, replace localhost by 0.0.0.0, see travis-ci/travis-ci#1304 --- tests/test_jobmanager.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/test_jobmanager.py b/tests/test_jobmanager.py index 26cc3db..357bcba 100644 --- a/tests/test_jobmanager.py +++ b/tests/test_jobmanager.py @@ -18,8 +18,9 @@ sys.path = [split(dirname(abspath(__file__)))[0]] + sys.path from jobmanager import jobmanager, progress -PORT = 42525 AUTHKEY = 'testing' +PORT = 42525 +SERVER = "0.0.0.0" def test_Signal_to_SIG_IGN(): @@ -136,7 +137,7 @@ def start_server(n, read_old_state=False, verbose=1): def start_client(verbose=1): print("START CLIENT") - jm_client = jobmanager.JobManager_Client(server = 'localhost', + jm_client = jobmanager.JobManager_Client(server = SERVER, authkey = AUTHKEY, port = PORT, nproc = 0, @@ -385,7 +386,7 @@ def test_check_fail(): time.sleep(1) print("START CLIENT") - jm_client = Client_Random_Error(server='localhost', + jm_client = Client_Random_Error(server=SERVER, authkey=AUTHKEY, port=PORT, nproc=0, @@ -561,7 +562,7 @@ def test_client_status(): return os.getpid() - client = Client_With_Status(server = 'localhost', + client = Client_With_Status(server = SERVER, authkey = AUTHKEY, port = PORT, nproc = 4, @@ -643,7 +644,7 @@ def test_shared_const_arg(): print(os.getpid(), arg, const_arg) return None - client = myClient(server='localhost', + client = myClient(server=SERVER, authkey=AUTHKEY, port = PORT, nproc=1, @@ -681,7 +682,7 @@ def test_digest_rejected(): return os.getpid() - client = Client_With_Status(server = 'localhost', + client = Client_With_Status(server = SERVER, authkey = AUTHKEY+' not the same', port = PORT, nproc = 4, @@ -742,7 +743,7 @@ def test_exception(): try: try: - autoproxy_connect(server='localhost', port=port, authkey=authkey) + autoproxy_connect(server=SERVER, port=port, authkey=authkey) except jobmanager.RemoteValueError: if (sys.version_info[0] == 3) and (p_version_server == 2): print("that is ok") # the occurrence of this Exception is normal @@ -761,20 +762,20 @@ def test_exception(): continue try: - autoproxy_connect(server='localhost', port=port+1, authkey=authkey) + autoproxy_connect(server=SERVER, port=port+1, authkey=authkey) except jobmanager.JMConnectionRefusedError: print("that is ok") except: raise try: - autoproxy_connect(server='localhost', port=port, authkey=authkey+'_') + autoproxy_connect(server=SERVER, port=port, authkey=authkey+'_') except jobmanager.AuthenticationError: print("that is ok") except: raise - m = autoproxy_connect(server='localhost', port=port, authkey=authkey) + m = autoproxy_connect(server=SERVER, port=port, authkey=authkey) q = m.get_q() From 84244fdc4ef78bc0de83dacd523c13faf9415045 Mon Sep 17 00:00:00 2001 From: paulmueller Date: Tue, 12 May 2015 10:48:22 +0200 Subject: [PATCH 20/56] install missing modules --- .travis.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a4a40f0..849e000 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,11 @@ python: before_install: # We need to use conda, because virtualenv did not work on travis with 3.4 - sudo apt-get update -qq -- if [[ $TRAVIS_PYTHON_VERSION == "2.7" ]]; then sudo apt-get -y install python-numpy; fi -- if [[ $TRAVIS_PYTHON_VERSION == "3.4" ]]; then sudo apt-get -y install python3-numpy; fi +install: +- if [[ $TRAVIS_PYTHON_VERSION == "2.7" ]]; then sudo apt-get -y install python-numpy python-scipy; fi +- if [[ $TRAVIS_PYTHON_VERSION == "3.4" ]]; then sudo apt-get -y install python3-numpy python3-scipy; fi +- pip install sqlitedict +- pip install psutil - pip freeze script: - travis_wait python setup.py test From b2ed1768278527a88af0458c70c09d9c0aad295f Mon Sep 17 00:00:00 2001 From: paulmueller Date: Tue, 12 May 2015 11:00:12 +0200 Subject: [PATCH 21/56] travis: system-site-packages --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 849e000..ce6f276 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,14 @@ language: python python: - "3.4" +virtualenv: + system_site_packages: true before_install: -# We need to use conda, because virtualenv did not work on travis with 3.4 - sudo apt-get update -qq +- if [[ $TRAVIS_PYTHON_VERSION == "2.7" ]]; then sudo apt-get -y install python-numpy python-scipy python-psutil; fi +- if [[ $TRAVIS_PYTHON_VERSION == "3.4" ]]; then sudo apt-get -y install python3-numpy python3-scipy python-psutil; fi install: -- if [[ $TRAVIS_PYTHON_VERSION == "2.7" ]]; then sudo apt-get -y install python-numpy python-scipy; fi -- if [[ $TRAVIS_PYTHON_VERSION == "3.4" ]]; then sudo apt-get -y install python3-numpy python3-scipy; fi - pip install sqlitedict -- pip install psutil - pip freeze script: - travis_wait python setup.py test From bcc734eedde31956c34a31fb5e23e9ba80fbf3a1 Mon Sep 17 00:00:00 2001 From: paulmueller Date: Tue, 12 May 2015 11:01:47 +0200 Subject: [PATCH 22/56] travis_minor changes --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ce6f276..8916708 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,6 @@ language: python +notifications: + email: false python: - "3.4" virtualenv: @@ -8,7 +10,7 @@ before_install: - if [[ $TRAVIS_PYTHON_VERSION == "2.7" ]]; then sudo apt-get -y install python-numpy python-scipy python-psutil; fi - if [[ $TRAVIS_PYTHON_VERSION == "3.4" ]]; then sudo apt-get -y install python3-numpy python3-scipy python-psutil; fi install: -- pip install sqlitedict +- travis_retry pip install sqlitedict - pip freeze script: - travis_wait python setup.py test From 84c0c87e16d850f3cb6ab2d593273a2b4bf5e9bc Mon Sep 17 00:00:00 2001 From: paulmueller Date: Tue, 12 May 2015 11:32:25 +0200 Subject: [PATCH 23/56] travis with miniconda --- .travis.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8916708..48f875d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,14 +3,19 @@ notifications: email: false python: - "3.4" -virtualenv: - system_site_packages: true before_install: - sudo apt-get update -qq -- if [[ $TRAVIS_PYTHON_VERSION == "2.7" ]]; then sudo apt-get -y install python-numpy python-scipy python-psutil; fi -- if [[ $TRAVIS_PYTHON_VERSION == "3.4" ]]; then sudo apt-get -y install python3-numpy python3-scipy python-psutil; fi +- sudo apt-get install -qq libatlas-dev libatlas-base-dev liblapack-dev gfortran +- wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh +- chmod +x miniconda.sh +- ./miniconda.sh -b +- export PATH=/home/travis/miniconda/bin:$PATH +- conda update --yes conda +- travis_retry conda install --yes python=$TRAVIS_PYTHON_VERSION pip numpy scipy matplotlib + install: - travis_retry pip install sqlitedict +- travis_retry pip install psutil - pip freeze script: - travis_wait python setup.py test From 138bcdd9c9648de696aad9173d238753b7ad7445 Mon Sep 17 00:00:00 2001 From: paulmueller Date: Tue, 12 May 2015 12:05:18 +0200 Subject: [PATCH 24/56] travis debug --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 48f875d..314481f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,11 +12,12 @@ before_install: - export PATH=/home/travis/miniconda/bin:$PATH - conda update --yes conda - travis_retry conda install --yes python=$TRAVIS_PYTHON_VERSION pip numpy scipy matplotlib - install: - travis_retry pip install sqlitedict - travis_retry pip install psutil - pip freeze script: +- netstat -tulpen +- lsof -i :42525 - travis_wait python setup.py test From da2378905a035218b2677ac106aa4699505bb331 Mon Sep 17 00:00:00 2001 From: paulmueller Date: Tue, 12 May 2015 13:00:29 +0200 Subject: [PATCH 25/56] wild hacking: 0.0.0.0 did not work --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.travis.yml b/.travis.yml index 314481f..c047019 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,12 @@ notifications: email: false python: - "3.4" +services: + - mysql + - rabbitmq + - mongodb + - memcached + - redis-server before_install: - sudo apt-get update -qq - sudo apt-get install -qq libatlas-dev libatlas-base-dev liblapack-dev gfortran From e7312b396977579ec6de9a5ac09685452ec88979 Mon Sep 17 00:00:00 2001 From: paulmueller Date: Tue, 12 May 2015 14:07:48 +0200 Subject: [PATCH 26/56] hostname from socket --- .travis.yml | 7 ------- tests/test_jobmanager.py | 3 ++- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index c047019..570a985 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,12 +3,6 @@ notifications: email: false python: - "3.4" -services: - - mysql - - rabbitmq - - mongodb - - memcached - - redis-server before_install: - sudo apt-get update -qq - sudo apt-get install -qq libatlas-dev libatlas-base-dev liblapack-dev gfortran @@ -24,6 +18,5 @@ install: - pip freeze script: - netstat -tulpen -- lsof -i :42525 - travis_wait python setup.py test diff --git a/tests/test_jobmanager.py b/tests/test_jobmanager.py index 357bcba..0ecdebe 100644 --- a/tests/test_jobmanager.py +++ b/tests/test_jobmanager.py @@ -9,6 +9,7 @@ import signal import multiprocessing as mp import numpy as np import traceback +import socket import subprocess import signal @@ -20,7 +21,7 @@ from jobmanager import jobmanager, progress AUTHKEY = 'testing' PORT = 42525 -SERVER = "0.0.0.0" +SERVER = socket.gethostname() def test_Signal_to_SIG_IGN(): From c00863f39d5a444c6f130b0665433a04bf591a69 Mon Sep 17 00:00:00 2001 From: paulmueller Date: Tue, 12 May 2015 14:37:29 +0200 Subject: [PATCH 27/56] increment ports for testing, threw errors on Ubuntu 14.04 and travis --- tests/test_clients.py | 2 +- tests/test_jobmanager.py | 35 ++++++++++++++++++++++++++++++++--- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/tests/test_clients.py b/tests/test_clients.py index 5248aa2..b78e2ba 100644 --- a/tests/test_clients.py +++ b/tests/test_clients.py @@ -126,7 +126,7 @@ def test_distributed_mathieu(): with jm.JobManager_Local(client_class = jm.clients.Integration_Client_REAL, authkey = authkey, - port = 42525, + port = 42520, const_arg = const_arg, nproc=1, verbose_client=2, diff --git a/tests/test_jobmanager.py b/tests/test_jobmanager.py index 0ecdebe..510d1af 100644 --- a/tests/test_jobmanager.py +++ b/tests/test_jobmanager.py @@ -25,6 +25,8 @@ SERVER = socket.gethostname() def test_Signal_to_SIG_IGN(): + global PORT + PORT += 1 def f(): jobmanager.Signal_to_SIG_IGN() print("before sleep") @@ -58,6 +60,8 @@ def test_Signal_to_SIG_IGN(): print("[+] terminated") def test_Signal_to_sys_exit(): + global PORT + PORT += 1 def f(): jobmanager.Signal_to_sys_exit() while True: @@ -93,6 +97,8 @@ def test_Signal_to_sys_exit(): print("[+] terminated") def test_Signal_to_terminate_process_list(): + global PORT + PORT += 1 def child_proc(): jobmanager.Signal_to_sys_exit() try: @@ -153,6 +159,8 @@ def test_jobmanager_basic(): check if all arguments are found in final_result of dump """ + global PORT + PORT += 1 n = 10 p_server = mp.Process(target=start_server, args=(n,)) p_server.start() @@ -185,6 +193,8 @@ def test_jobmanager_basic(): def test_jobmanager_server_signals(): + global PORT + PORT += 1 print("## TEST SIGTERM ##") p_server = mp.Process(target=start_server, args=(30,)) p_server.start() @@ -208,7 +218,7 @@ def test_jobmanager_server_signals(): assert len(ref_set - args_set) == 0 print("[+] args_set from dump contains all arguments") - + PORT += 1 print("## TEST SIGINT ##") p_server = mp.Process(target=start_server, args=(30,)) p_server.start() @@ -245,6 +255,8 @@ def test_shutdown_server_while_client_running(): check if the final_result and the args dump end up to include all arguments given """ + global PORT + PORT += 1 n = 1000 @@ -253,6 +265,7 @@ def test_shutdown_server_while_client_running(): time.sleep(1) + PORT += 1 p_client = mp.Process(target=start_client, args=(2,)) p_client.start() @@ -315,7 +328,8 @@ def shutdown_client(sig): if server does not terminate on time, something must be wrong with args_set check if the final_result contain all arguments given """ - + global PORT + PORT += 1 n = 300 print("## terminate client with {} ##".format(progress.signal_dict[sig])) @@ -368,6 +382,8 @@ def shutdown_client(sig): print("[+] all arguments found in final_results") def test_check_fail(): + global PORT + PORT += 1 class Client_Random_Error(jobmanager.JobManager_Client): def func(self, args, const_args, c, m): c.value = 0 @@ -440,6 +456,8 @@ def test_jobmanager_read_old_stat(): check if all arguments are found in final_result of dump """ + global PORT + PORT += 1 n = 100 p_server = mp.Process(target=start_server, args=(n,)) p_server.start() @@ -461,7 +479,7 @@ def test_jobmanager_read_old_stat(): print("[+] client and server terminated") time.sleep(2) - + PORT += 1 p_server = mp.Process(target=start_server, args=(n,True)) p_server.start() @@ -548,6 +566,8 @@ def test_hashedViewOnNumpyArray(): assert bh2 in s def test_client_status(): + global PORT + PORT += 1 n = 10 p_server = mp.Process(target=start_server, args=(n,False,0)) p_server.start() @@ -572,6 +592,8 @@ def test_client_status(): p_server.join() def test_jobmanager_local(): + global PORT + PORT += 1 args = range(1,200) with jobmanager.JobManager_Local(client_class = jobmanager.JobManager_Client, authkey = AUTHKEY, @@ -583,6 +605,8 @@ def test_jobmanager_local(): jm_server.start() def test_start_server_on_used_port(): + global PORT + PORT += 1 def start_server(): const_arg = None arg = [10,20,30] @@ -625,6 +649,8 @@ def test_start_server_on_used_port(): assert not other_error def test_shared_const_arg(): + global PORT + PORT += 1 def start_server(): const_arg = {1:1, 2:2, 3:3} arg = [10,20,30] @@ -653,6 +679,7 @@ def test_shared_const_arg(): client.start() + PORT += 1 p1 = mp.Process(target=start_server) p2 = mp.Process(target=start_client) @@ -668,6 +695,8 @@ def test_shared_const_arg(): p1.join() def test_digest_rejected(): + global PORT + PORT += 1 n = 10 p_server = mp.Process(target=start_server, args=(n,False,0)) p_server.start() From fd7f190962365354766cc0c0af3a42c1d5009af0 Mon Sep 17 00:00:00 2001 From: paulmueller Date: Tue, 12 May 2015 14:48:50 +0200 Subject: [PATCH 28/56] more deterministic port selection --- tests/test_jobmanager.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_jobmanager.py b/tests/test_jobmanager.py index 510d1af..bd77964 100644 --- a/tests/test_jobmanager.py +++ b/tests/test_jobmanager.py @@ -725,7 +725,9 @@ def test_digest_rejected(): p_server.join() -def test_exception(): +def test_exception(): + global PORT + PORT += 1 class MyManager_Client(jobmanager.BaseManager): pass @@ -760,7 +762,8 @@ def test_exception(): return m for p_version_server in [2, 3]: - port = np.random.randint(20000, 30000) + PORT += 10 + port = PORT authkey = 'q' with open("ap_server.out", 'w') as outfile: From fff8f58a2a116960e7afc4c25822056c5217f96c Mon Sep 17 00:00:00 2001 From: Richard Hartmann Date: Tue, 19 May 2015 19:35:34 +0200 Subject: [PATCH 29/56] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b8104e6..2838b0e 100644 --- a/README.md +++ b/README.md @@ -26,3 +26,5 @@ Running an example Running tests python setup.py test + +[travis test] From bff8f6534bbff335930855b1fa763f5a74ecc0db Mon Sep 17 00:00:00 2001 From: Richard Hartmann Date: Wed, 27 May 2015 10:29:16 +0200 Subject: [PATCH 30/56] added missing file for teting #24 --- tests/start_autoproxy_server.py | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 tests/start_autoproxy_server.py diff --git a/tests/start_autoproxy_server.py b/tests/start_autoproxy_server.py new file mode 100644 index 0000000..de0e896 --- /dev/null +++ b/tests/start_autoproxy_server.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from __future__ import division, print_function + +import sys +import time +import multiprocessing as mp +import numpy as np +import traceback + +from os.path import abspath, dirname, split +# Add parent directory to beginning of path variable +sys.path = [split(dirname(abspath(__file__)))[0]] + sys.path + +from jobmanager import jobmanager, progress + +jobmanager.Signal_to_sys_exit(verbose=2) + +class MyManager_Server(jobmanager.BaseManager): + pass + + +port = int(sys.argv[1]) +authkey = sys.argv[2] + + +q = jobmanager.myQueue() +MyManager_Server.register('get_q', callable = lambda: q) +m = MyManager_Server(address = ('', port), authkey = bytearray(authkey, encoding='utf8')) +m.start() +print("Server started (manager process at {})".format(m._process.pid)) +sys.stdout.flush() + +try: + while True: + time.sleep(10) + print(time.time()) + sys.stdout.flush() +except: + print("Exception") + sys.stdout.flush() + +finally: +# print("stop server ...") +# pm = m._process +# m.shutdown() +# progress.check_process_termination(proc=pm, +# identifier="autoproxy manager subprocess", +# timeout=2, +# verbose=2, +# auto_kill_on_last_resort=True) + print("Server stopped") \ No newline at end of file From 9ba0627dcd0ac73589bd4d7427e126a40ea1edd3 Mon Sep 17 00:00:00 2001 From: Richard Hartmann Date: Thu, 28 May 2015 22:45:09 +0200 Subject: [PATCH 31/56] still try to fix the non passing test_exception in jobmanager tests (see #24) --- .travis.yml | 1 + jobmanager/jobmanager.py | 1 + tests/test_jobmanager.py | 62 ++++++++++++++++++++++++++++------------ 3 files changed, 45 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index 570a985..81b35ac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ notifications: email: false python: - "3.4" +- "2.7" before_install: - sudo apt-get update -qq - sudo apt-get install -qq libatlas-dev libatlas-base-dev liblapack-dev gfortran diff --git a/jobmanager/jobmanager.py b/jobmanager/jobmanager.py index e2adccc..c3fd2b9 100644 --- a/jobmanager/jobmanager.py +++ b/jobmanager/jobmanager.py @@ -1438,6 +1438,7 @@ def handler_broken_pipe_error(e, verbose): def handler_connection_refused(e, identifier, dest, verbose): if verbose > 1: print("this usually means that no matching Manager object was instanciated at destination side!") + print("either there is no Manager running at all, or it is listening to another port.") raise JMConnectionRefusedError(e) def handler_connection_reset(identifier, dest, c, reconnect_wait, reconnect_tries, verbose): diff --git a/tests/test_jobmanager.py b/tests/test_jobmanager.py index bd77964..cbb1e61 100644 --- a/tests/test_jobmanager.py +++ b/tests/test_jobmanager.py @@ -748,7 +748,10 @@ def test_exception(): str(port), authkey] + print("#"*40) + print("start an autoproxy server with command") print(cmd) + print("#"*40) return subprocess.Popen(cmd, env=python_env, stdout=outfile, stderr=subprocess.STDOUT) def autoproxy_connect(server, port, authkey): @@ -757,7 +760,7 @@ def test_exception(): m = MyManager_Client(address = (server, port), authkey = bytearray(authkey, encoding='utf8')) - jobmanager.call_connect(m.connect, dest = jobmanager.address_authkey_from_manager(m), verbose=1) + jobmanager.call_connect(m.connect, dest = jobmanager.address_authkey_from_manager(m), verbose=2) return m @@ -771,45 +774,68 @@ def test_exception(): print("autoproxy server running with PID {}".format(p_server.pid)) time.sleep(1) - print("running tests ...") + print("running tests with python {} ...".format(sys.version_info[0])) print() try: - try: - autoproxy_connect(server=SERVER, port=port, authkey=authkey) - except jobmanager.RemoteValueError: - if (sys.version_info[0] == 3) and (p_version_server == 2): - print("that is ok") # the occurrence of this Exception is normal - pass - else: - raise # reraise exception - except ValueError: - if (sys.version_info[0] == 2) and (p_version_server == 3): - print("that is ok") # the occurrence of this Exception is normal - pass - else: - raise # reraise exception + if sys.version_info[0] == 3: + print("we are using python 3 ... try to connect ...") + try: + autoproxy_connect(server=SERVER, port=port, authkey=authkey) + except jobmanager.RemoteValueError as e: + if p_version_server == 2: + print("that is ok, because the server is running on python2") # the occurrence of this Exception is normal + print() + pass + else: + print("RemoteValueError error") + raise # reraise exception + except Exception as e: + print("unexpected error {}".format(e)) + raise + + elif sys.version_info[0] == 2: + print("we are using python 2 ... try to connect ...") + try: + autoproxy_connect(server=SERVER, port=port, authkey=authkey) + except ValueError as e: + if p_version_server == 3: + print("that is ok, because the server is running on python3") # the occurrence of this Exception is normal + print() + pass + else: + print("JMConnectionRefusedError error") + raise # reraise exception + except Exception as e: + print("unexpected error {}".format(e)) + raise # all the following only for the same python versions if (sys.version_info[0] != p_version_server): continue try: + print("try to connect to server, use wrong port") autoproxy_connect(server=SERVER, port=port+1, authkey=authkey) except jobmanager.JMConnectionRefusedError: print("that is ok") + print() except: raise try: + print("try to connect to server, use wrong authkey") autoproxy_connect(server=SERVER, port=port, authkey=authkey+'_') except jobmanager.AuthenticationError: print("that is ok") + print() except: raise m = autoproxy_connect(server=SERVER, port=port, authkey=authkey) + print("try pass some data forth and back ...") + q = m.get_q() q_get = jobmanager.proxy_operation_decorator_python3(q, 'get') @@ -819,9 +845,7 @@ def test_exception(): q_put(s1) s2 = q_get() - assert s1 == s2 - - + assert s1 == s2 finally: From 029d64928f2ebf216b07991b1275c96d9d580815 Mon Sep 17 00:00:00 2001 From: Richard Hartmann Date: Fri, 29 May 2015 09:51:17 +0200 Subject: [PATCH 32/56] print more debug info on Manager_server to see whats going wrong when travis runs test_exception --- jobmanager/jobmanager.py | 4 +- tests/start_autoproxy_server.py | 19 ++------- tests/test_jobmanager.py | 76 +++++++++++++++++---------------- 3 files changed, 45 insertions(+), 54 deletions(-) diff --git a/jobmanager/jobmanager.py b/jobmanager/jobmanager.py index c3fd2b9..935ef96 100644 --- a/jobmanager/jobmanager.py +++ b/jobmanager/jobmanager.py @@ -1335,7 +1335,7 @@ def call_connect_python3(connect, dest, verbose=1, identifier='', reconnect_wait else: # no exception if verbose > 1: - print("{}connection to {} successfully stablished".format(identifier, dest)) + print("{}connection to {} successfully established".format(identifier, dest)) return True def call_connect_python2(connect, dest, verbose=1, identifier='', reconnect_wait=2, reconnect_tries=3): @@ -1378,7 +1378,7 @@ def call_connect_python2(connect, dest, verbose=1, identifier='', reconnect_wait else: # no exception if verbose > 1: - print("{}connection to {} successfully stablished".format(identifier, dest)) + print("{}connection to {} successfully established".format(identifier, dest)) return True # SUCCESS -> return True call_connect = call_connect_python2 if sys.version_info[0] == 2 else call_connect_python3 diff --git a/tests/start_autoproxy_server.py b/tests/start_autoproxy_server.py index de0e896..76fc50d 100644 --- a/tests/start_autoproxy_server.py +++ b/tests/start_autoproxy_server.py @@ -28,25 +28,14 @@ q = jobmanager.myQueue() MyManager_Server.register('get_q', callable = lambda: q) m = MyManager_Server(address = ('', port), authkey = bytearray(authkey, encoding='utf8')) m.start() -print("Server started (manager process at {})".format(m._process.pid)) +print("MyManager_Server started (manager process at {})".format(m._process.pid)) sys.stdout.flush() try: while True: time.sleep(10) - print(time.time()) + print("MyManager_Server is running, time:", time.time()) sys.stdout.flush() -except: - print("Exception") - sys.stdout.flush() - finally: -# print("stop server ...") -# pm = m._process -# m.shutdown() -# progress.check_process_termination(proc=pm, -# identifier="autoproxy manager subprocess", -# timeout=2, -# verbose=2, -# auto_kill_on_last_resort=True) - print("Server stopped") \ No newline at end of file + print("MyManager_Server stopping") + sys.stdout.flush() \ No newline at end of file diff --git a/tests/test_jobmanager.py b/tests/test_jobmanager.py index cbb1e61..3434126 100644 --- a/tests/test_jobmanager.py +++ b/tests/test_jobmanager.py @@ -748,10 +748,10 @@ def test_exception(): str(port), authkey] - print("#"*40) + print("+"*40) print("start an autoproxy server with command") print(cmd) - print("#"*40) + print("+"*40) return subprocess.Popen(cmd, env=python_env, stdout=outfile, stderr=subprocess.STDOUT) def autoproxy_connect(server, port, authkey): @@ -768,16 +768,15 @@ def test_exception(): PORT += 10 port = PORT authkey = 'q' - with open("ap_server.out", 'w') as outfile: - - p_server = autoproxy_server(p_version_server, port, authkey, outfile) - print("autoproxy server running with PID {}".format(p_server.pid)) - time.sleep(1) - - print("running tests with python {} ...".format(sys.version_info[0])) - print() - - try: + try: + with open("ap_server.out", 'w') as outfile: + p_server = autoproxy_server(p_version_server, port, authkey, outfile) + print("autoproxy server running with PID {}".format(p_server.pid)) + time.sleep(1) + + print("running tests with python {} ...".format(sys.version_info[0])) + print() + if sys.version_info[0] == 3: print("we are using python 3 ... try to connect ...") try: @@ -786,7 +785,6 @@ def test_exception(): if p_version_server == 2: print("that is ok, because the server is running on python2") # the occurrence of this Exception is normal print() - pass else: print("RemoteValueError error") raise # reraise exception @@ -802,7 +800,6 @@ def test_exception(): if p_version_server == 3: print("that is ok, because the server is running on python3") # the occurrence of this Exception is normal print() - pass else: print("JMConnectionRefusedError error") raise # reraise exception @@ -846,29 +843,34 @@ def test_exception(): s2 = q_get() assert s1 == s2 - - - finally: - print() - print("tests done! terminate server ...".format()) - - p_server.send_signal(signal.SIGTERM) - - t = time.time() - timeout = 10 - r = None - while r is None: - r = p_server.poll() - time.sleep(1) - print("will kill server in {:.1f}s".format(timeout - (time.time() - t))) - if (time.time() - t) > timeout: - print("timeout exceeded, kill p_server") - print("the managers subprocess will still be running, and needs to be killed by hand") - p_server.send_signal(signal.SIGKILL) - break - - print("server terminated with exitcode {}".format(r)) - + + finally: + print() + print("tests done! terminate server ...".format()) + + p_server.send_signal(signal.SIGTERM) + + t = time.time() + timeout = 10 + r = None + while r is None: + r = p_server.poll() + time.sleep(1) + print("will kill server in {:.1f}s".format(timeout - (time.time() - t))) + if (time.time() - t) > timeout: + print("timeout exceeded, kill p_server") + print("the managers subprocess will still be running, and needs to be killed by hand") + p_server.send_signal(signal.SIGKILL) + break + + print("server terminated with exitcode {}".format(r)) + + with open("ap_server.out", 'r') as outfile: + print("+"*40) + print("this is the server output:") + for l in outfile: + print(" {}".format(l[:-1])) + print("+"*40) From 6ea374595c03f767f7260c2d95e94e87ce93fc05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Fri, 29 May 2015 10:21:13 +0200 Subject: [PATCH 33/56] allow failures for python 2.7 --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 81b35ac..2ec0983 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,9 @@ notifications: python: - "3.4" - "2.7" +matrix: + allow_failures: + - python: "2.7" before_install: - sudo apt-get update -qq - sudo apt-get install -qq libatlas-dev libatlas-base-dev liblapack-dev gfortran From af917e327e4b535b29fcf3c85c00bd3d67eb8398 Mon Sep 17 00:00:00 2001 From: cimatosa Date: Mon, 15 Jun 2015 16:13:21 +0200 Subject: [PATCH 34/56] create path in case it does not exists, persistentData --- jobmanager/persistentData.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jobmanager/persistentData.py b/jobmanager/persistentData.py index bc62aaa..fc866f2 100644 --- a/jobmanager/persistentData.py +++ b/jobmanager/persistentData.py @@ -44,7 +44,10 @@ class PersistentDataStructure(object): self._name = name self._path = abspath(path) if not exists(self._path): - raise RuntimeError("given path does not exists ({} -> {})".format(path, self._path)) + print("given path does not exists ({} -> {})".format(path, self._path)) + print("create path") + os.makedirs(self._path) + self.verbose = verbose From 5e630d5bcd1522e450eece2d7a41090f70256875 Mon Sep 17 00:00:00 2001 From: cimatosa Date: Tue, 23 Jun 2015 19:50:37 +0200 Subject: [PATCH 35/56] added minor missing debug info --- jobmanager/jobmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobmanager/jobmanager.py b/jobmanager/jobmanager.py index 935ef96..da9df48 100644 --- a/jobmanager/jobmanager.py +++ b/jobmanager/jobmanager.py @@ -1257,7 +1257,7 @@ class Signal_to_terminate_process_list(object): def _handler(self, signal, frame): if self.verbose > 0: - print(": received sig {} -> terminate all given subprocesses".format(self.identifier, progress.signal_dict[signal])) + print("{}: received sig {} -> terminate all given subprocesses".format(self.identifier, progress.signal_dict[signal])) for i, p in enumerate(self.process_list): p.terminate() progress.check_process_termination(proc = p, From d5a4c8ea9fa8e609d7ca81069ff6b124897572a8 Mon Sep 17 00:00:00 2001 From: cimatosa Date: Tue, 23 Jun 2015 19:50:46 +0200 Subject: [PATCH 36/56] testing --- tests/test_decorators.py | 6 +++++- tests/test_jobmanager.py | 21 +++++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/test_decorators.py b/tests/test_decorators.py index b12912a..b96eb1a 100644 --- a/tests/test_decorators.py +++ b/tests/test_decorators.py @@ -62,8 +62,9 @@ def my_func_ProgressBarOverrideCount(c = None, m = None): c.value = i + @decorators.ProgressBar -def test_decorated_func_calls_decorated_func( +def testing_decorated_func_calls_decorated_func( c = decorators.progress.UnsignedIntValue(val=0), m = decorators.progress.UnsignedIntValue(val=1), ): @@ -82,6 +83,9 @@ def test_decorated_func_calls_decorated_func( # anything - which makes the output pretty. _my_func_1(arg=i, kwarg=0, sleep=0.005) +def test_decorated_func_calls_decorated_func(): + testing_decorated_func_calls_decorated_func() + def test_decorator(): c = progress.UnsignedIntValue(val=0) m = progress.UnsignedIntValue(val=100) diff --git a/tests/test_jobmanager.py b/tests/test_jobmanager.py index 3434126..3ea0c72 100644 --- a/tests/test_jobmanager.py +++ b/tests/test_jobmanager.py @@ -113,8 +113,8 @@ def test_Signal_to_terminate_process_list(): for i in range(n): p.append(mp.Process(target=child_proc)) p[-1].start() - - jobmanager.Signal_to_terminate_process_list(p) + + jobmanager.Signal_to_terminate_process_list(identifier='mother_proc', process_list=p, identifier_list=["proc_{}".format(i+1) for i in range(n)], verbose=2) print("spawned {} processes".format(n)) for i in range(n): p[i].join() @@ -122,7 +122,8 @@ def test_Signal_to_terminate_process_list(): p_mother = mp.Process(target=mother_proc) p_mother.start() - time.sleep(0.5) + time.sleep(1.5) + assert p_mother.is_alive() print("send SIGINT") os.kill(p_mother.pid, signal.SIGINT) @@ -879,13 +880,13 @@ if __name__ == "__main__": pass else: func = [ -# test_Signal_to_SIG_IGN, -# test_Signal_to_sys_exit, -# test_Signal_to_terminate_process_list, + test_Signal_to_SIG_IGN, + test_Signal_to_sys_exit, + test_Signal_to_terminate_process_list, # -# test_jobmanager_basic, -# test_jobmanager_server_signals, -# test_shutdown_server_while_client_running, + test_jobmanager_basic, + test_jobmanager_server_signals, + test_shutdown_server_while_client_running, # test_shutdown_client, # test_check_fail, # test_jobmanager_read_old_stat, @@ -896,7 +897,7 @@ if __name__ == "__main__": # test_start_server_on_used_port, # test_shared_const_arg, # test_digest_rejected, - test_exception, +# test_exception, lambda : print("END") ] From 0fa9244cda331d87e46a04f70e0fd3286b8d145c Mon Sep 17 00:00:00 2001 From: cimatosa Date: Fri, 26 Jun 2015 17:01:05 +0200 Subject: [PATCH 37/56] added optional info line --- jobmanager/jobmanager.py | 18 ++++++---- jobmanager/progress.py | 76 ++++++++++++++++++++++++++++------------ tests/test_progress.py | 30 ++++++++++++---- 3 files changed, 88 insertions(+), 36 deletions(-) diff --git a/jobmanager/jobmanager.py b/jobmanager/jobmanager.py index da9df48..a893ebb 100644 --- a/jobmanager/jobmanager.py +++ b/jobmanager/jobmanager.py @@ -1069,18 +1069,22 @@ class JobManager_Server(object): else: Progress = progress.ProgressSilentDummy - with Progress(count = self._numresults, - max_count = self._numjobs, - interval = self.msg_interval, - speed_calc_cycles=self.speed_calc_cycles, - verbose = self.verbose, - sigint='ign', - sigterm='ign') as stat: + info_line = progress.StringValue(num_of_bytes=80) + + with Progress(count = self._numresults, + max_count = self._numjobs, + interval = self.msg_interval, + speed_calc_cycles = self.speed_calc_cycles, + verbose = self.verbose, + sigint = 'ign', + sigterm = 'ign', + info_line = info_line) as stat: stat.start() while (len(self.args_set) - self.fail_q.qsize()) > 0: try: + info_line.value = "result_q size: {}".format(self.result_q.qsize()).encode('utf-8') arg, result = self.result_q.get(timeout=1) self.args_set.remove(arg) self.numresults = self.numjobs - len(self.args_set) diff --git a/jobmanager/progress.py b/jobmanager/progress.py index 600a5f6..6d78ad4 100644 --- a/jobmanager/progress.py +++ b/jobmanager/progress.py @@ -331,16 +331,17 @@ class Progress(Loop): use of the 'with' statement is highly encouraged. """ def __init__(self, - count, - max_count=None, - prepend=None, - width='auto', - speed_calc_cycles=10, - interval=1, - verbose=0, - sigint='stop', - sigterm='stop', - name='progress'): + count, + max_count = None, + prepend = None, + width = 'auto', + speed_calc_cycles = 10, + interval = 1, + verbose = 0, + sigint = 'stop', + sigterm = 'stop', + name = 'progress', + info_line = None): """ count [mp.Value] - shared memory to hold the current state, (list or single value) @@ -382,7 +383,7 @@ class Progress(Loop): except TypeError: raise TypeError("'max_count' must be iterable") else: - assert isinstance(max_count, mp.sharedctypes.Synchronized), "'max_count' must be if the type multiprocessing.sharedctypes.Synchronized" + assert isinstance(max_count, mp.sharedctypes.Synchronized), "'max_count' must be of the type multiprocessing.sharedctypes.Synchronized" max_count = [max_count] else: max_count = [None] * self.len @@ -429,6 +430,8 @@ class Progress(Loop): self.show_on_exit = False self.add_args = {} + self.info_line = info_line + # setup loop class with func super(Progress, self).__init__(func=Progress.show_stat_wrapper_multi, args=(self.count, @@ -444,7 +447,8 @@ class Progress(Loop): self.__class__.show_stat, self.len, self.add_args, - self.lock), + self.lock, + self.info_line), interval=interval, verbose=verbose, sigint=sigint, @@ -551,7 +555,9 @@ class Progress(Loop): self.__class__.show_stat, self.len, self.add_args, - self.lock) + self.lock, + self.info_line, + no_move_up=True) def reset(self, i = None): """ @@ -629,7 +635,9 @@ class Progress(Loop): show_stat_function, len_, add_args, - lock): + lock, + info_line, + no_move_up=False): """ call the static method show_stat_wrapper for each process """ @@ -650,7 +658,22 @@ class Progress(Loop): add_args, i, lock[i]) - print(ESC_MOVE_LINE_UP(len_) + ESC_NO_CHAR_ATTR, end='') + n = len_ + if info_line is not None: + s = info_line.value.decode('utf-8') + s = s.split('\n') + n += len(s) + for si in s: + if width == 'auto': + width = get_terminal_width() + if len(si) > width: + si = si[:width] + print("{0:<{1}}".format(si, width)) + + if no_move_up: + n = 0 + + print(ESC_MOVE_LINE_UP(n) + ESC_NO_CHAR_ATTR, end='') sys.stdout.flush() def start(self): @@ -683,7 +706,7 @@ class Progress(Loop): if self.show_on_exit: self._show_stat() - print('\n'*(self.len-1)) + print() self.show_on_exit = False @@ -701,7 +724,8 @@ class ProgressBar(Progress): verbose=0, sigint='stop', sigterm='stop', - name='progress_bar'): + name='progress_bar', + info_line=None): """ width [int/'auto'] - the number of characters used to show the Progress bar, use 'auto' to determine width from terminal information -> see _set_width @@ -715,7 +739,8 @@ class ProgressBar(Progress): verbose = verbose, sigint=sigint, sigterm=sigterm, - name=name) + name=name, + info_line=info_line) self._PRE_PREPEND = ESC_NO_CHAR_ATTR + ESC_RED self._POST_PREPEND = ESC_BOLD + ESC_GREEN @@ -782,7 +807,8 @@ class ProgressBarCounter(Progress): verbose=0, sigint='stop', sigterm='stop', - name='progress_bar_counter'): + name='progress_bar_counter', + info_line=None): super(ProgressBarCounter, self).__init__(count=count, max_count=max_count, @@ -793,7 +819,8 @@ class ProgressBarCounter(Progress): verbose = verbose, sigint=sigint, sigterm=sigterm, - name=name) + name=name, + info_line=info_line) self.counter_count = [] self.counter_q = [] @@ -889,7 +916,8 @@ class ProgressBarFancy(Progress): verbose=0, sigint='stop', sigterm='stop', - name='progress_bar'): + name='progress_bar', + info_line=None): """ width [int/'auto'] - the number of characters used to show the Progress bar, use 'auto' to determine width from terminal information -> see _set_width @@ -906,7 +934,8 @@ class ProgressBarFancy(Progress): verbose = verbose, sigint=sigint, sigterm=sigterm, - name=name) + name=name, + info_line=info_line) @staticmethod def get_d(s1, s2, width, lp, lps): @@ -1180,6 +1209,9 @@ def FloatValue(val=0.): def UnsignedIntValue(val=0): return mp.Value('I', val, lock=True) +def StringValue(num_of_bytes): + return mp.Array('c', bytearray(num_of_bytes), lock=True) + def check_process_termination(proc, identifier, timeout, verbose=0, auto_kill_on_last_resort = False): proc.join(timeout) diff --git a/tests/test_progress.py b/tests/test_progress.py index 2ebd2fa..2f7f7a1 100644 --- a/tests/test_progress.py +++ b/tests/test_progress.py @@ -730,7 +730,22 @@ def test_progress_bar_counter_fancy_hide_bar(): time.sleep(0.0432) if (time.time() - t0) > 15: - break + break + +def test_info_line(): + c1 = progress.UnsignedIntValue(val=0) + s = progress.StringValue(80) + m1 = progress.UnsignedIntValue(val=30) + + with progress.ProgressBarFancy(count=c1, max_count=m1, verbose=1, interval=0.2, info_line=s) as sc: + sc.start() + while True: + c1.value = c1.value + 1 + if c1.value > 10: + s.value = b'info_line\nline2' + time.sleep(0.1) + if c1.value >= m1.value: + break if __name__ == "__main__": func = [ @@ -752,12 +767,13 @@ if __name__ == "__main__": # test_progress_bar_counter_hide_bar, # test_progress_bar_slow_change, # test_progress_bar_start_stop, - test_progress_bar_fancy, - test_progress_bar_multi_fancy, - test_progress_bar_fancy_small, - test_progress_bar_counter_fancy, - test_progress_bar_counter_fancy_non_max, - test_progress_bar_counter_fancy_hide_bar, +# test_progress_bar_fancy, +# test_progress_bar_multi_fancy, +# test_progress_bar_fancy_small, +# test_progress_bar_counter_fancy, +# test_progress_bar_counter_fancy_non_max, +# test_progress_bar_counter_fancy_hide_bar, + test_info_line, lambda: print("END") ] From e9cc57b3e6cee7288f5e3fb0ff0f49bc3f4b1139 Mon Sep 17 00:00:00 2001 From: cimatosa Date: Mon, 10 Aug 2015 15:22:33 +0200 Subject: [PATCH 38/56] minor change --- jobmanager/servers.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/jobmanager/servers.py b/jobmanager/servers.py index a9a8284..31adfc8 100644 --- a/jobmanager/servers.py +++ b/jobmanager/servers.py @@ -85,7 +85,16 @@ class PersistentData_Server(JobManager_Server): print("{}: overwriting existing data is DISABLED".format(self._identifier)) def process_new_result(self, arg, result): - self.pds[data_as_binary_key(arg.id)] = (arg, result) + """ + use arg.id as key to store the pair (arg, result) in the data base + + return True, if a new data set was added (key not already in pds) + otherwise false + """ + key = data_as_binary_key(arg.id) + has_key = key in self.pds + self.pds[key] = (arg, result) + return not has_key def put_arg(self, a): a_bin = data_as_binary_key(a.id) From 7a566cbb62029c4dacd8f8d24327f40232b59b24 Mon Sep 17 00:00:00 2001 From: Travis CI Date: Mon, 24 Aug 2015 18:31:34 +0200 Subject: [PATCH 39/56] travis: update documentation after every build --- .travis.yml | 19 +++++++--- doc/commit_gh-pages.py | 59 ------------------------------- doc/deploy_ghpages.py | 80 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 64 deletions(-) delete mode 100644 doc/commit_gh-pages.py create mode 100644 doc/deploy_ghpages.py diff --git a/.travis.yml b/.travis.yml index 2ec0983..6a3f726 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,18 +1,22 @@ language: python +env: + global: + - GH_REF: github.com/cimatosa/jobmanager.git + - secure: Q3YVqRTs3QLspdzHgYMMuOoMypn7yetkb7v0LFe/68AwxZAwEkpCF24FHL/cEH9khc99AGbDIST81undY6kJLWLEDHYIWaoKUbC8OmaT7H6MUdfLtznsv1NnOLut9Z3So/mTMGQHCmTga9NKsuFpDMxg/oxp3wS915ar6+PLh3w= notifications: email: false python: -- "3.4" -- "2.7" +- '3.4' +- '2.7' matrix: allow_failures: - - python: "2.7" + - python: '2.7' before_install: - sudo apt-get update -qq - sudo apt-get install -qq libatlas-dev libatlas-base-dev liblapack-dev gfortran - wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh - chmod +x miniconda.sh -- ./miniconda.sh -b +- "./miniconda.sh -b" - export PATH=/home/travis/miniconda/bin:$PATH - conda update --yes conda - travis_retry conda install --yes python=$TRAVIS_PYTHON_VERSION pip numpy scipy matplotlib @@ -23,4 +27,9 @@ install: script: - netstat -tulpen - travis_wait python setup.py test - +after_success: +- git config credential.helper "store --file=.git/credentials" +- echo "https://${GH_TOKEN}:@github.com" > .git/credentials +- if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry conda install --yes sphinx; fi +- if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then pip install numpydoc; fi +- if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then python doc/deploy_ghpages.py; fi diff --git a/doc/commit_gh-pages.py b/doc/commit_gh-pages.py deleted file mode 100644 index 253bb30..0000000 --- a/doc/commit_gh-pages.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -from __future__ import print_function -import os -from os.path import abspath,dirname -import sys -import subprocess as sp - - -os.chdir(dirname(dirname(abspath(__file__)))) - - -def checkout_master(stashed): - # go back to master - sp.check_output(["git", 'checkout', 'master']) - - if stashed: - print("Applying saved stash.") - # get last stash? - sp.check_output(["git", 'stash', 'apply']) - - -def checkout_ghpages(): - # checkout the gh-pages branch - try: - sp.check_output(["git", 'checkout', 'gh-pages']) - except: - print("Automatically stashing current changes.") - sp.check_output(["git", 'stash']) - stashed = True - sp.check_output(["git", 'checkout', 'gh-pages']) - else: - stashed = False - - return stashed - -stashed = checkout_ghpages() - - -# copy built files -if os.system("cp -r ./build/sphinx/html/* ./") != 0: - checkout_master(stashed) - sys.exit() - -for item in os.listdir("./build/sphinx/html/"): - # Make sure we have added all files from html - if not item.startswith("."): - os.system("git add ./{}".format(item)) - -# commit changes -if len(sp.check_output(["git", "diff", "HEAD"]).strip()) > 0: - sp.check_output(["git", 'commit', '-a', '-m', '"automated doc upload"']) - -# push -try: - sp.check_output(["git", 'push']) -except: - print("Could not push to gh-pages.") - -checkout_master(stashed) diff --git a/doc/deploy_ghpages.py b/doc/deploy_ghpages.py new file mode 100644 index 0000000..8875aa0 --- /dev/null +++ b/doc/deploy_ghpages.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Publish the documentation on GitHub Pages. + +Prerequisites +------------- + +1. Create empty gh-pages branch: + + git branch gh-pages + git checkout gh-pages + git symbolic-ref HEAD refs/heads/gh-pages + rm .git/index + git clean -fdx + + +2. Setup sphinx. + + python setup.py build_sphinx + + should create a build/sphinx/html folder in the repository root. + + +3. Create GitHub repo token and encrypt it + + gem install travis + travis encrypt GH_TOKEN="" --add + + +4. Add the encrypted token to .travis.yml + + env: + global: + - GH_REF: github.com//.git + - secure: "jdcn3kM/dI0zvVTn0UKgal8Br+745Qc1plaKXHcoKhwcwN+0Q1y5H1BnaF0KV2dWWeExVXMpqQMLOCylUSUmd30+hFqUgd3gFQ+oh9pF/+N72uzjnxHAyVjai5Lh7QnjN0SLCd2/xLYwaUIHjWbWsr5t2vK9UuyphZ6/F+7OHf+u8BErviE9HUunD7u4Q2XRaUF0oHuF8stoWbJgnQZtUZFr+qS1Gc3vF6/KBkMqjnq/DgBV61cWsnVUS1HVak/sGClPRXZMSGyz8d63zDxfA5NDO6AbPVgK02k+QV8KQCyIX7of8rBvBmWkBYGw5RnaeETLIAf6JrCKMiQzlJQZiMyLUvd/WflSIBKJyr5YmUKCjFkwvbKKvCU3WBUxFT2p7trKZip5JWg37OMvOAO8eiatf2FC1klNly1KHADU88QqNoi/0y2R/a+1Csrl8Gr/lXZkW4mMkI2due9epLwccDJtMF8Xc39EqRR46xA7Lx9vy7szYW5lLux3zwx1tH40wV6/dX4ZVFoWp/zfJw7TKdOHuOwjZuOuKp/shfJs94G9YCu7bBtvrGv9qCH2KiSgm1NJviwcsZWsVHaq1nP0LliDE7EM3Q0mnkYzlvfOOhA2G5Ka3rHl1RFj7+WYzO5GaAFWU7piP/kdBwc0Mu+hb6PMoy0oeLt39BDr29bNKMs=" + +5. Add the deploy command to .travis.yml + + after_success: + - git config credential.helper "store --file=.git/credentials" + - echo "https://${GH_TOKEN}:@github.com" > .git/credentials + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then pip install numpydoc sphinx; fi + - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then python doc/deploy_ghpages.py; fi + +""" +from __future__ import print_function +import os +from os.path import dirname, abspath +import subprocess as sp + + +# go to root of repository +os.chdir(dirname(dirname(abspath(__file__)))) + +# build sphinx +sp.check_output("python setup.py build_sphinx", shell=True) + +# clone into new folder the gh-pages branch +sp.check_output("git config --global user.email 'travis@example.com'", shell=True) +sp.check_output("git config --global user.name 'Travis CI'", shell=True) +sp.check_output("git config --global credential.helper 'store --file=.git/credentials'", shell=True) +sp.check_output("echo 'https://${GH_TOKEN}:@github.com' > .git/credentials", shell=True) +sp.check_output("git clone --depth 1 -b gh-pages https://${GH_TOKEN}@${GH_REF} gh_pages", shell=True) + +# copy everything from ./build/sphinx/html to ./gh_pages +#sp.check_output("cp -r ./build/sphinx/html/* ./gh_pages/", shell=True) +sp.check_output("rsync -rt --del --exclude='.git' --exclude='.nojekyll' ./build/sphinx/html/* ./gh_pages/", shell=True) + +# commit changes +os.chdir("gh_pages") +sp.check_output("echo 'https://${GH_TOKEN}:@github.com' > .git/credentials", shell=True) +sp.check_output("git add --all ./*", shell=True) +try: + sp.check_output("git commit -a -m 'travis bot build ${TRAVIS_COMMIT} [ci skip]'", shell=True) +except: + pass #nothing to do +else: + sp.check_output("git push --force --quiet origin gh-pages", shell=True) + \ No newline at end of file From 1c07b0f394857a495c1ff188e0b9cf6ea29cb7a6 Mon Sep 17 00:00:00 2001 From: Travis CI Date: Mon, 24 Aug 2015 19:21:35 +0200 Subject: [PATCH 40/56] travis miniconda testing suite --- .travis.yml | 27 +++++++++++++++++++++------ tests/test_jobmanager.py | 6 ++++-- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6a3f726..3bffdc0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,20 +12,35 @@ matrix: allow_failures: - python: '2.7' before_install: -- sudo apt-get update -qq -- sudo apt-get install -qq libatlas-dev libatlas-base-dev liblapack-dev gfortran - wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh - chmod +x miniconda.sh - "./miniconda.sh -b" - export PATH=/home/travis/miniconda/bin:$PATH - conda update --yes conda -- travis_retry conda install --yes python=$TRAVIS_PYTHON_VERSION pip numpy scipy matplotlib +## The following lines are necessary for a test that needs Python 2 and 3 +# Install python2 and python3 anaconda (required for testing) +# python2 +- conda create -y -n my_py2 python=2 pip numpy scipy matplotlib +- source activate my_py2 +- pip install sqlitedict psutil +- source deactivate +# python3 +- conda create -y -n my_py3 python=3 pip numpy scipy matplotlib +- source activate my_py3 +- pip install sqlitedict psutil +- source deactivate +# add the two environments to the path +- export PATH=/home/travis/miniconda/envs/my_py3/bin:$PATH +- export PATH=/home/travis/miniconda/envs/my_py2/bin:$PATH +# activate the correct environment +- if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then source activate my_py2; fi +- if [[ $TRAVIS_PYTHON_VERSION == 3.4 ]]; then source activate my_py3; fi +- which python +- which python2.7 +- which python3.4 install: -- travis_retry pip install sqlitedict -- travis_retry pip install psutil - pip freeze script: -- netstat -tulpen - travis_wait python setup.py test after_success: - git config credential.helper "store --file=.git/credentials" diff --git a/tests/test_jobmanager.py b/tests/test_jobmanager.py index 3ea0c72..3eacfb2 100644 --- a/tests/test_jobmanager.py +++ b/tests/test_jobmanager.py @@ -733,12 +733,14 @@ def test_exception(): pass def autoproxy_server(which_python, port, authkey, outfile): + python_env = {"PYTHONPATH": os.path.dirname(os.__file__)} + # env will be + # "/usr/lib/python2.7" for python 2 + # "/usr/lib/python3.4" for python 3 if which_python == 2: python_interpreter = "python2.7" - python_env = {"PYTHONPATH": "/usr/lib/python2.7"} elif which_python == 3: python_interpreter = "python3.4" - python_env = {"PYTHONPATH": "/usr/lib/python3.4"} else: raise ValueError("'which_python' must be 2 or 3") From a96948e7c877cdbd7115d53ecc8758f68a52c5c3 Mon Sep 17 00:00:00 2001 From: Travis CI Date: Mon, 24 Aug 2015 19:53:57 +0200 Subject: [PATCH 41/56] connection refused? --- tests/test_jobmanager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_jobmanager.py b/tests/test_jobmanager.py index 3eacfb2..7a33656 100644 --- a/tests/test_jobmanager.py +++ b/tests/test_jobmanager.py @@ -768,6 +768,7 @@ def test_exception(): return m for p_version_server in [2, 3]: + global PORT PORT += 10 port = PORT authkey = 'q' From 5502cc60829ab41caea731492b373635275db822 Mon Sep 17 00:00:00 2001 From: Travis CI Date: Mon, 24 Aug 2015 19:58:18 +0200 Subject: [PATCH 42/56] syntax --- tests/test_jobmanager.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_jobmanager.py b/tests/test_jobmanager.py index 7a33656..81ddae5 100644 --- a/tests/test_jobmanager.py +++ b/tests/test_jobmanager.py @@ -728,7 +728,6 @@ def test_digest_rejected(): def test_exception(): global PORT - PORT += 1 class MyManager_Client(jobmanager.BaseManager): pass @@ -768,8 +767,7 @@ def test_exception(): return m for p_version_server in [2, 3]: - global PORT - PORT += 10 + PORT += 2 # plus two because we also check for wrong port port = PORT authkey = 'q' try: From b0d00e5695b36026d09c0826b1bdb5a9a6717de1 Mon Sep 17 00:00:00 2001 From: Travis CI Date: Mon, 24 Aug 2015 20:39:17 +0200 Subject: [PATCH 43/56] pythonpath not correctly set --- .travis.yml | 16 ++++++++-------- tests/test_jobmanager.py | 5 +++++ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3bffdc0..d0edf14 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,21 +20,21 @@ before_install: ## The following lines are necessary for a test that needs Python 2 and 3 # Install python2 and python3 anaconda (required for testing) # python2 -- conda create -y -n my_py2 python=2 pip numpy scipy matplotlib -- source activate my_py2 +- conda create -y -n my_py2.7 python=2 pip numpy scipy matplotlib +- source activate my_py2.7 - pip install sqlitedict psutil - source deactivate # python3 -- conda create -y -n my_py3 python=3 pip numpy scipy matplotlib -- source activate my_py3 +- conda create -y -n my_py3.4 python=3 pip numpy scipy matplotlib +- source activate my_py3.4 - pip install sqlitedict psutil - source deactivate # add the two environments to the path -- export PATH=/home/travis/miniconda/envs/my_py3/bin:$PATH -- export PATH=/home/travis/miniconda/envs/my_py2/bin:$PATH +- export PATH=/home/travis/miniconda/envs/my_py3.4/bin:$PATH +- export PATH=/home/travis/miniconda/envs/my_py2.7/bin:$PATH # activate the correct environment -- if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then source activate my_py2; fi -- if [[ $TRAVIS_PYTHON_VERSION == 3.4 ]]; then source activate my_py3; fi +- if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then source activate my_py2.7; fi +- if [[ $TRAVIS_PYTHON_VERSION == 3.4 ]]; then source activate my_py3.4; fi - which python - which python2.7 - which python3.4 diff --git a/tests/test_jobmanager.py b/tests/test_jobmanager.py index 81ddae5..8eb8dcc 100644 --- a/tests/test_jobmanager.py +++ b/tests/test_jobmanager.py @@ -738,14 +738,17 @@ def test_exception(): # "/usr/lib/python3.4" for python 3 if which_python == 2: python_interpreter = "python2.7" + python_env = python_env.replace("3.4", "2.7") elif which_python == 3: python_interpreter = "python3.4" + python_env = python_env.replace("2.7", "3.4") else: raise ValueError("'which_python' must be 2 or 3") path = dirname(abspath(__file__)) cmd = [python_interpreter, + "{}/start_autoproxy_server.py".format(path), str(port), authkey] @@ -753,6 +756,8 @@ def test_exception(): print("+"*40) print("start an autoproxy server with command") print(cmd) + print("and environment") + print(python_env) print("+"*40) return subprocess.Popen(cmd, env=python_env, stdout=outfile, stderr=subprocess.STDOUT) From 7142c96081dfcd31a924a8d4e4e4bc5b1ad08fe8 Mon Sep 17 00:00:00 2001 From: Travis CI Date: Mon, 24 Aug 2015 21:09:58 +0200 Subject: [PATCH 44/56] UnboundLocalError: local variable 'p_server' referenced before assignment --- tests/test_jobmanager.py | 65 ++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/tests/test_jobmanager.py b/tests/test_jobmanager.py index 8eb8dcc..ceb9a73 100644 --- a/tests/test_jobmanager.py +++ b/tests/test_jobmanager.py @@ -775,12 +775,11 @@ def test_exception(): PORT += 2 # plus two because we also check for wrong port port = PORT authkey = 'q' - try: - with open("ap_server.out", 'w') as outfile: - p_server = autoproxy_server(p_version_server, port, authkey, outfile) - print("autoproxy server running with PID {}".format(p_server.pid)) - time.sleep(1) - + with open("ap_server.out", 'w') as outfile: + p_server = autoproxy_server(p_version_server, port, authkey, outfile) + print("autoproxy server running with PID {}".format(p_server.pid)) + time.sleep(1) + try: print("running tests with python {} ...".format(sys.version_info[0])) print() @@ -851,34 +850,34 @@ def test_exception(): assert s1 == s2 - finally: - print() - print("tests done! terminate server ...".format()) - - p_server.send_signal(signal.SIGTERM) - - t = time.time() - timeout = 10 - r = None - while r is None: - r = p_server.poll() - time.sleep(1) - print("will kill server in {:.1f}s".format(timeout - (time.time() - t))) - if (time.time() - t) > timeout: - print("timeout exceeded, kill p_server") - print("the managers subprocess will still be running, and needs to be killed by hand") - p_server.send_signal(signal.SIGKILL) - break - - print("server terminated with exitcode {}".format(r)) - - with open("ap_server.out", 'r') as outfile: - print("+"*40) - print("this is the server output:") - for l in outfile: - print(" {}".format(l[:-1])) - print("+"*40) + finally: + print() + print("tests done! terminate server ...".format()) + + p_server.send_signal(signal.SIGTERM) + + t = time.time() + timeout = 10 + r = None + while r is None: + r = p_server.poll() + time.sleep(1) + print("will kill server in {:.1f}s".format(timeout - (time.time() - t))) + if (time.time() - t) > timeout: + print("timeout exceeded, kill p_server") + print("the managers subprocess will still be running, and needs to be killed by hand") + p_server.send_signal(signal.SIGKILL) + break + + print("server terminated with exitcode {}".format(r)) + with open("ap_server.out", 'r') as outfile: + print("+"*40) + print("this is the server output:") + for l in outfile: + print(" {}".format(l[:-1])) + print("+"*40) + if __name__ == "__main__": From 9d81673c7d75a42b5a288787f89d651d9212cf2c Mon Sep 17 00:00:00 2001 From: Travis CI Date: Mon, 24 Aug 2015 21:28:51 +0200 Subject: [PATCH 45/56] replace string --- tests/test_jobmanager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_jobmanager.py b/tests/test_jobmanager.py index ceb9a73..2d42246 100644 --- a/tests/test_jobmanager.py +++ b/tests/test_jobmanager.py @@ -732,20 +732,20 @@ def test_exception(): pass def autoproxy_server(which_python, port, authkey, outfile): - python_env = {"PYTHONPATH": os.path.dirname(os.__file__)} + envpath = os.path.dirname(os.__file__) # env will be # "/usr/lib/python2.7" for python 2 # "/usr/lib/python3.4" for python 3 if which_python == 2: python_interpreter = "python2.7" - python_env = python_env.replace("3.4", "2.7") + envpath = envpath.replace("3.4", "2.7") elif which_python == 3: python_interpreter = "python3.4" - python_env = python_env.replace("2.7", "3.4") + envpath = envpath.replace("2.7", "3.4") else: raise ValueError("'which_python' must be 2 or 3") - + python_env = {"PYTHONPATH": envpath} path = dirname(abspath(__file__)) cmd = [python_interpreter, From 45d8237d944797824df876004607841a285cc527 Mon Sep 17 00:00:00 2001 From: Travis CI Date: Mon, 24 Aug 2015 21:48:24 +0200 Subject: [PATCH 46/56] setting env variables --- .travis.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.travis.yml b/.travis.yml index d0edf14..0b7bef1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,6 +41,17 @@ before_install: install: - pip freeze script: +- which python +- which python2.7 +- which python3.4 +- export PATH=/home/travis/miniconda/envs/my_py3.4/bin:$PATH +- export PATH=/home/travis/miniconda/envs/my_py2.7/bin:$PATH +# activate the correct environment +- if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then source activate my_py2.7; fi +- if [[ $TRAVIS_PYTHON_VERSION == 3.4 ]]; then source activate my_py3.4; fi +- which python +- which python2.7 +- which python3.4 - travis_wait python setup.py test after_success: - git config credential.helper "store --file=.git/credentials" From dca7cc9db42f5fdcc6d59a16703dc4bbaf47c207 Mon Sep 17 00:00:00 2001 From: Travis CI Date: Mon, 24 Aug 2015 22:11:55 +0200 Subject: [PATCH 47/56] debug --- tests/start_autoproxy_server.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/start_autoproxy_server.py b/tests/start_autoproxy_server.py index 76fc50d..2072331 100644 --- a/tests/start_autoproxy_server.py +++ b/tests/start_autoproxy_server.py @@ -2,7 +2,14 @@ # -*- coding: utf-8 -*- from __future__ import division, print_function +import os import sys + +# debug: +print(sys.executable) +print(sys.path) +print(os.environ) + import time import multiprocessing as mp import numpy as np @@ -15,7 +22,7 @@ sys.path = [split(dirname(abspath(__file__)))[0]] + sys.path from jobmanager import jobmanager, progress jobmanager.Signal_to_sys_exit(verbose=2) - + class MyManager_Server(jobmanager.BaseManager): pass @@ -38,4 +45,4 @@ try: sys.stdout.flush() finally: print("MyManager_Server stopping") - sys.stdout.flush() \ No newline at end of file + sys.stdout.flush() From 40307f8ee4b6f8af1d3dc59a3471a06e60c39501 Mon Sep 17 00:00:00 2001 From: Travis CI Date: Mon, 24 Aug 2015 22:21:11 +0200 Subject: [PATCH 48/56] add site-packages --- tests/test_jobmanager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_jobmanager.py b/tests/test_jobmanager.py index 2d42246..cba06c0 100644 --- a/tests/test_jobmanager.py +++ b/tests/test_jobmanager.py @@ -732,7 +732,8 @@ def test_exception(): pass def autoproxy_server(which_python, port, authkey, outfile): - envpath = os.path.dirname(os.__file__) + libpath = os.path.dirname(os.__file__) + envpath = "{LIB}:{LIB}/site-packages".format(LIB=libpath) # env will be # "/usr/lib/python2.7" for python 2 # "/usr/lib/python3.4" for python 3 From c378cb8ebaa68f2421cdd3aa6c266c62d4a8b137 Mon Sep 17 00:00:00 2001 From: Travis CI Date: Mon, 24 Aug 2015 22:46:49 +0200 Subject: [PATCH 49/56] add other lib locations --- tests/test_jobmanager.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_jobmanager.py b/tests/test_jobmanager.py index cba06c0..abf58e8 100644 --- a/tests/test_jobmanager.py +++ b/tests/test_jobmanager.py @@ -734,6 +734,10 @@ def test_exception(): def autoproxy_server(which_python, port, authkey, outfile): libpath = os.path.dirname(os.__file__) envpath = "{LIB}:{LIB}/site-packages".format(LIB=libpath) + envpath += ":{LIB}/lib-old".format(LIB=libpath) + envpath += ":{LIB}/lib-tk".format(LIB=libpath) + envpath += ":{LIB}/lib-dynload".format(LIB=libpath) + envpath += ":{LIB}/plat-linux2".format(LIB=libpath) # env will be # "/usr/lib/python2.7" for python 2 # "/usr/lib/python3.4" for python 3 From 64f462882de8c557bbe93bb35b508686790e1780 Mon Sep 17 00:00:00 2001 From: Travis CI Date: Mon, 24 Aug 2015 23:25:55 +0200 Subject: [PATCH 50/56] patch hashlib --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 0b7bef1..6390fb8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,6 +32,9 @@ before_install: # add the two environments to the path - export PATH=/home/travis/miniconda/envs/my_py3.4/bin:$PATH - export PATH=/home/travis/miniconda/envs/my_py2.7/bin:$PATH +# hack to circumvent errors in tests +- sed -i -- 's/except ImportError:/except:/g' /home/travis/miniconda/envs/my_py3.4/lib/python2.7/hashlib.py +- sed -i -- 's/except ImportError:/except:/g' /home/travis/miniconda/envs/my_py3.4/lib/python3.4/hashlib.py # activate the correct environment - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then source activate my_py2.7; fi - if [[ $TRAVIS_PYTHON_VERSION == 3.4 ]]; then source activate my_py3.4; fi From 49a3f39a667c5d3932699e3a535dd01014ddc0db Mon Sep 17 00:00:00 2001 From: Travis CI Date: Mon, 24 Aug 2015 23:28:21 +0200 Subject: [PATCH 51/56] typo --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6390fb8..4f48044 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,7 @@ before_install: - export PATH=/home/travis/miniconda/envs/my_py3.4/bin:$PATH - export PATH=/home/travis/miniconda/envs/my_py2.7/bin:$PATH # hack to circumvent errors in tests -- sed -i -- 's/except ImportError:/except:/g' /home/travis/miniconda/envs/my_py3.4/lib/python2.7/hashlib.py +- sed -i -- 's/except ImportError:/except:/g' /home/travis/miniconda/envs/my_py2.7/lib/python2.7/hashlib.py - sed -i -- 's/except ImportError:/except:/g' /home/travis/miniconda/envs/my_py3.4/lib/python3.4/hashlib.py # activate the correct environment - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then source activate my_py2.7; fi From c706db28a87fad596b713614449733e4a4abbbea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Tue, 25 Aug 2015 09:34:41 +0200 Subject: [PATCH 52/56] copy host environment for test --- .travis.yml | 1 + tests/test_jobmanager.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4f48044..fbbcf2e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ matrix: allow_failures: - python: '2.7' before_install: +- sudo apt-get install openssl - wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh - chmod +x miniconda.sh - "./miniconda.sh -b" diff --git a/tests/test_jobmanager.py b/tests/test_jobmanager.py index abf58e8..9dc9bf9 100644 --- a/tests/test_jobmanager.py +++ b/tests/test_jobmanager.py @@ -733,6 +733,7 @@ def test_exception(): def autoproxy_server(which_python, port, authkey, outfile): libpath = os.path.dirname(os.__file__) + python_env = os.environ.copy() envpath = "{LIB}:{LIB}/site-packages".format(LIB=libpath) envpath += ":{LIB}/lib-old".format(LIB=libpath) envpath += ":{LIB}/lib-tk".format(LIB=libpath) @@ -750,7 +751,7 @@ def test_exception(): else: raise ValueError("'which_python' must be 2 or 3") - python_env = {"PYTHONPATH": envpath} + python_env["PYTHONPATH"] = envpath path = dirname(abspath(__file__)) cmd = [python_interpreter, From 566c959f19bb3ccaabee4f18b8c2b3dbda7fa957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Tue, 25 Aug 2015 09:39:54 +0200 Subject: [PATCH 53/56] apt-get update --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index fbbcf2e..ffaeb9c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ matrix: allow_failures: - python: '2.7' before_install: +- sudo apt-get update -qq -y - sudo apt-get install openssl - wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh - chmod +x miniconda.sh From 0abaefeb1e8a3ef6704c5fa9fd168e68d7bbe8db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20M=C3=BCller?= Date: Tue, 25 Aug 2015 09:54:27 +0200 Subject: [PATCH 54/56] remove debug-print messages --- .travis.yml | 15 +-------------- tests/start_autoproxy_server.py | 5 ----- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index ffaeb9c..630123b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ matrix: allow_failures: - python: '2.7' before_install: +# required for testing - sudo apt-get update -qq -y - sudo apt-get install openssl - wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh @@ -34,9 +35,6 @@ before_install: # add the two environments to the path - export PATH=/home/travis/miniconda/envs/my_py3.4/bin:$PATH - export PATH=/home/travis/miniconda/envs/my_py2.7/bin:$PATH -# hack to circumvent errors in tests -- sed -i -- 's/except ImportError:/except:/g' /home/travis/miniconda/envs/my_py2.7/lib/python2.7/hashlib.py -- sed -i -- 's/except ImportError:/except:/g' /home/travis/miniconda/envs/my_py3.4/lib/python3.4/hashlib.py # activate the correct environment - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then source activate my_py2.7; fi - if [[ $TRAVIS_PYTHON_VERSION == 3.4 ]]; then source activate my_py3.4; fi @@ -46,17 +44,6 @@ before_install: install: - pip freeze script: -- which python -- which python2.7 -- which python3.4 -- export PATH=/home/travis/miniconda/envs/my_py3.4/bin:$PATH -- export PATH=/home/travis/miniconda/envs/my_py2.7/bin:$PATH -# activate the correct environment -- if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then source activate my_py2.7; fi -- if [[ $TRAVIS_PYTHON_VERSION == 3.4 ]]; then source activate my_py3.4; fi -- which python -- which python2.7 -- which python3.4 - travis_wait python setup.py test after_success: - git config credential.helper "store --file=.git/credentials" diff --git a/tests/start_autoproxy_server.py b/tests/start_autoproxy_server.py index 2072331..d520ba0 100644 --- a/tests/start_autoproxy_server.py +++ b/tests/start_autoproxy_server.py @@ -5,11 +5,6 @@ from __future__ import division, print_function import os import sys -# debug: -print(sys.executable) -print(sys.path) -print(os.environ) - import time import multiprocessing as mp import numpy as np From 662fb2032b2f7499ec4f76377cb239df79addedf Mon Sep 17 00:00:00 2001 From: Travis CI Date: Tue, 25 Aug 2015 10:14:41 +0200 Subject: [PATCH 55/56] automated doc upload --- .travis.yml | 6 ++-- doc/Readme.md | 9 ------ doc/deploy_ghpages.py | 6 ++-- jobmanager/decorators.py | 62 ++++++++++++++++++++-------------------- 4 files changed, 37 insertions(+), 46 deletions(-) diff --git a/.travis.yml b/.travis.yml index 630123b..f2e5216 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,6 +48,6 @@ script: after_success: - git config credential.helper "store --file=.git/credentials" - echo "https://${GH_TOKEN}:@github.com" > .git/credentials -- if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then travis_retry conda install --yes sphinx; fi -- if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then pip install numpydoc; fi -- if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then python doc/deploy_ghpages.py; fi +- if [[ $TRAVIS_PYTHON_VERSION == 3.4 ]]; then travis_retry conda install --yes sphinx; fi +- if [[ $TRAVIS_PYTHON_VERSION == 3.4 ]]; then pip install numpydoc; fi +- if [[ $TRAVIS_PYTHON_VERSION == 3.4 ]]; then python doc/deploy_ghpages.py; fi diff --git a/doc/Readme.md b/doc/Readme.md index 976ec85..a585922 100644 --- a/doc/Readme.md +++ b/doc/Readme.md @@ -6,12 +6,3 @@ Install [numpydoc](https://pypi.python.org/pypi/numpydoc): To compile the documentation, run python setup.py build_sphinx - - -To upload the documentation to gh-pages, run - - python setup.py commit_doc - -or - - python doc/commit_gh-pages.py diff --git a/doc/deploy_ghpages.py b/doc/deploy_ghpages.py index 8875aa0..6cfa320 100644 --- a/doc/deploy_ghpages.py +++ b/doc/deploy_ghpages.py @@ -40,8 +40,8 @@ Prerequisites after_success: - git config credential.helper "store --file=.git/credentials" - echo "https://${GH_TOKEN}:@github.com" > .git/credentials - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then pip install numpydoc sphinx; fi - - if [[ $TRAVIS_PYTHON_VERSION == 2.7 ]]; then python doc/deploy_ghpages.py; fi + - if [[ $TRAVIS_PYTHON_VERSION == 3.4 ]]; then pip install numpydoc sphinx; fi + - if [[ $TRAVIS_PYTHON_VERSION == 3.4 ]]; then python doc/deploy_ghpages.py; fi """ from __future__ import print_function @@ -77,4 +77,4 @@ except: pass #nothing to do else: sp.check_output("git push --force --quiet origin gh-pages", shell=True) - \ No newline at end of file + diff --git a/jobmanager/decorators.py b/jobmanager/decorators.py index f39c2db..5b3a1d4 100644 --- a/jobmanager/decorators.py +++ b/jobmanager/decorators.py @@ -158,38 +158,38 @@ class ProgressBar(object): class ProgressBarExtended(ProgressBar): """ - extends the ProgressBar such that - - on can turn of the ProgressBar by giving an extra argument, - namely 'progress_bar_off' and set its value to 'True'. - - further there will be an additional argument passed to the function - called 'progress_bar' which allows to stop the progress bar from - within the function. note that there will be an function signature error - if the function does not accept the extra argument 'progress_bar'. So a - general **kwargs at the end of the functions arguments will help. - That is also the reason why the extended version comes in an extra class - because it might otherwise break compatibility. - - Example - ------- - - >>> import jobmanager as jm - - >>> c = jm.progress.UnsignedIntValue(val=0) - >>> m = jm.progress.UnsignedIntValue(val=20) + extends the ProgressBar such that - >>> @jm.decorators.ProgressBarExtended # choose 'ProgressBarExtended' - >>> def my_func_kwargs(c, m, **kwargs): # simply add '**kwargs' here - >>> for i in range(m.value): - >>> c.value = i+1 - >>> time.sleep(0.1) - - >>> # same as when using ProgressBar - >>> my_func_kwargs(c, m) - - >>> # a simple kwarg will switch the progressBar off - >>> my_func_kwargs(c, m, progress_bar_off=True) + on can turn of the ProgressBar by giving an extra argument, + namely 'progress_bar_off' and set its value to 'True'. + + further there will be an additional argument passed to the function + called 'progress_bar' which allows to stop the progress bar from + within the function. note that there will be an function signature error + if the function does not accept the extra argument 'progress_bar'. So a + general **kwargs at the end of the functions arguments will help. + That is also the reason why the extended version comes in an extra class + because it might otherwise break compatibility. + + Example + ------- + + >>> import jobmanager as jm + + >>> c = jm.progress.UnsignedIntValue(val=0) + >>> m = jm.progress.UnsignedIntValue(val=20) + + >>> @jm.decorators.ProgressBarExtended # choose 'ProgressBarExtended' + >>> def my_func_kwargs(c, m, **kwargs): # simply add '**kwargs' here + >>> for i in range(m.value): + >>> c.value = i+1 + >>> time.sleep(0.1) + + >>> # same as when using ProgressBar + >>> my_func_kwargs(c, m) + + >>> # a simple kwarg will switch the progressBar off + >>> my_func_kwargs(c, m, progress_bar_off=True) """ def __call__(self, *args, **kwargs): # Bind the args and kwds to the argument names of self.func From 4a2c2ff562ebebb061f3d57a8d386350ad817b31 Mon Sep 17 00:00:00 2001 From: Travis CI Date: Tue, 25 Aug 2015 10:34:49 +0200 Subject: [PATCH 56/56] travis gh-pages: display commit --- doc/conf.py | 2 +- doc/deploy_ghpages.py | 3 ++- jobmanager/clients.py | 48 +++++++++++++++++++++---------------------- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 98bd5c8..c462f86 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -139,7 +139,7 @@ exclude_patterns = ['_build'] # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = 'classic' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/doc/deploy_ghpages.py b/doc/deploy_ghpages.py index 6cfa320..f276f08 100644 --- a/doc/deploy_ghpages.py +++ b/doc/deploy_ghpages.py @@ -71,8 +71,9 @@ sp.check_output("rsync -rt --del --exclude='.git' --exclude='.nojekyll' ./build/ os.chdir("gh_pages") sp.check_output("echo 'https://${GH_TOKEN}:@github.com' > .git/credentials", shell=True) sp.check_output("git add --all ./*", shell=True) + try: - sp.check_output("git commit -a -m 'travis bot build ${TRAVIS_COMMIT} [ci skip]'", shell=True) + sp.check_output("git commit -a -m 'travis bot build {} [ci skip]'".format(os.getenv("TRAVIS_COMMIT")), shell=True) except: pass #nothing to do else: diff --git a/jobmanager/clients.py b/jobmanager/clients.py index 38de94b..9f507a8 100644 --- a/jobmanager/clients.py +++ b/jobmanager/clients.py @@ -16,30 +16,30 @@ from . import ode_wrapper def merge_arg_and_const_arg(arg, const_arg): """ - prepares data from arg and const_arg such that they can be passed - to the general integration routine - - arg and const_arg are both assumed to be dictionaries - - the merge process must not alter arg nor const_arg - in order to be used in the jobmanager context - - returns the arguments passed to the function - defining the derivative such that - args_dgl = arg['args'] + const_arg['args'] - where as arg['args'] and const_arg['args'] have been assumed to be tuples - - e.g. - arg['args'] = (2, 'low') - const_arg['args'] = (15, np.pi) - f will be called with - f(t, x, 2, 'low', 15, np.pi) - - returns further the combined dictionary - arg + const_arg with the keyword 'args' removed - - For any duplicate keys the value will be the value - from the 'arg' dictionary. + prepares data from arg and const_arg such that they can be passed + to the general integration routine + + arg and const_arg are both assumed to be dictionaries + + the merge process must not alter arg nor const_arg + in order to be used in the jobmanager context + + returns the arguments passed to the function + defining the derivative such that + args_dgl = arg['args'] + const_arg['args'] + where as arg['args'] and const_arg['args'] have been assumed to be tuples + + e.g. + arg['args'] = (2, 'low') + const_arg['args'] = (15, np.pi) + f will be called with + f(t, x, 2, 'low', 15, np.pi) + + returns further the combined dictionary + arg + const_arg with the keyword 'args' removed + + For any duplicate keys the value will be the value + from the 'arg' dictionary. """ # allows arg to be a namedtuple (or any other object that