working on persistentData, seems to work now on first glance, including some tests

jobmanager/__init__.py

@@ -8,6 +8,7 @@ from . import clients
 from . import decorators
 from . import servers
 from . import progress
+from . import persistentData
 
 # ode_wrapper requires scipy
 try:

jobmanager/persistentData.py

@@ -1,21 +1,26 @@
 import sqlitedict as sqd
+from os.path import abspath, join, exists
 import os
 
 MAGIC_SIGN = 0xff4a87
 
 class PersistentDataStructure(object):
-    def __init__(self, name, path=""):
+    def __init__(self, name, path="./", verbose=0):
         self.__name = name
-        self.__path = path
+        self.__path = abspath(path)
+        if not exists(self.__path):
+            raise RuntimeError("given path does not exists ({} -> {})".format(path, self.__path))
+        
+        self.verbose = verbose
         
         # create directory to hold sub structures
-        dir_name = os.path.join(path, name)
+        self.__dir_name = join(self.__path, "__" + self.__name)
-        if not os.path.exists(dir_name):
+        if not exists(self.__dir_name):
-            os.mkdir(dir_name)
+            os.mkdir(self.__dir_name)
         
         # open actual sqltedict
-        file_name = os.path.join(path, name + '.db')
+        self.__filename = join(self.__dir_name, self.__name + '.db')
-        self.db = sqd.SqliteDict(filename = file_name, autocommit=True)
+        self.db = sqd.SqliteDict(filename = self.__filename, autocommit=True)
         
         if not 0 in self.db:
             self.db[0] = 1
@@ -24,12 +29,15 @@ class PersistentDataStructure(object):
         return self
     
     def __exit__(self, exc_type, exc_value, traceback):
-        print("exit called for '{}' in '{}'".format(self.__name, self.__path))
+        if self.verbose > 1:
+            print("exit called for '{}' in '{}'".format(self.__name, self.__dir_name))
         self.close()
         
     def close(self):
         self.db.close()
-        print("closed db '{}' in '{}'".format(self.__name, self.__path))
+        if self.verbose > 1:
+            print("closed db '{}' in '{}'".format(self.__name, self.__dir_name))
+            
        
     def __check_key(self, key):
         if key == 0:
@@ -37,6 +45,14 @@ class PersistentDataStructure(object):
         
         return True
     
+    def __is_sub_data(self, value):
+        try:
+            assert value['magic'] == MAGIC_SIGN
+            value.pop('magic')
+            return True
+        except:
+            return False
+    
     def has_key(self, key):
         return (key in self.db)
         
@@ -72,21 +88,19 @@ class PersistentDataStructure(object):
             db0 = self.db[0]
             new_name = "{}".format(db0)
             self.db[0] = db0+1
-            kwargs = {'name': new_name, 'path': os.path.join(self.__path,self.__name), 'magic': MAGIC_SIGN}
+            kwargs = {'name': new_name, 'path': self.__dir_name, 'magic': MAGIC_SIGN}
             self.db[key] = kwargs
             kwargs.pop('magic')
-            return PersistentDataStructure(**kwargs)
+            return PersistentDataStructure(verbose = self.verbose, **kwargs)
         else:
             raise RuntimeError("can NOT create new SubData, key already found!")
         
     def getData(self, key, create_sub_data = False):
         if key in self.db:
             value = self.db[key]
-            try:
+            if self.__is_sub_data(value):
-                assert value['magic'] == MAGIC_SIGN
-                value.pop('magic')
                 return PersistentDataStructure(**value)
-            except: 
+            else:
                 return value 
         else:
             if not create_sub_data:
@@ -94,8 +108,14 @@ class PersistentDataStructure(object):
             else:
                 return self.newSubData(key)
             
+    def __getitem__(self, key):
+        return self.db[key]
     
+    def __setitem__(self, key, value):
+        if key in self.db:
+            if self.__is_sub_data(self.db[key]):
+                raise RuntimeWarning("values which hold sub_data structures can not be overwritten!")
+                return None
             
-        
+        self.db[key] = value
- 
         

test/test_persistentData.py

@@ -1,12 +1,17 @@
-import os
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from __future__ import division, print_function
+
 import sys
+from os.path import abspath, dirname, split
 
-sys.path.append(os.path.dirname(__file__))
+# Add parent directory to beginning of path variable
+sys.path = [split(dirname(abspath(__file__)))[0]] + sys.path
 
-import persistentData as pd
+import jobmanager as jm
 
 def test_pd():
-    with pd.PersistentDataStructure(name='test_data') as data:
+    with jm.persistentData.PersistentDataStructure(name='test_data', verbose=2) as data:
         key = 'a'
         value = 1
         data.setData(key=key, value=value)
@@ -26,9 +31,37 @@ def test_pd():
                 assert sub_data.getData(key) == 3
                 assert data.getData(key) == 1
                 
+            with sub_data.getData(key_sub, create_sub_data=True) as sub_sub_data:
+                assert sub_sub_data.getData(key) == 4
+                assert sub_data.getData(key) == 3
+                assert data.getData(key) == 1
                 
+def test_pd_bytes():
+    import pickle
+    
+    t1 = (3.4, 4.5, 5.6, 6.7, 7.8, 8.9)
+    t2 = (3.4, 4.5, 5.6, 6.7, 7.8, 8.9, 9,1)
+    
+    b1 = pickle.dumps(t1)
+    b2 = pickle.dumps(t2)
+    
+    with jm.persistentData.PersistentDataStructure(name='base') as base_data:
+        with base_data.getData(key=b1, create_sub_data=True) as sub_data:
+            for i in range(10):
+                sub_data[i] = t2
+                
+        base_data[b2] = t1
+        
+    
+    with jm.persistentData.PersistentDataStructure(name='base') as base_data:
+        with base_data.getData(key=b1, create_sub_data=True) as sub_data:
+            for i in range(10):
+                assert sub_data[i] == t2
+        
+        assert base_data[b2] == t1
 
    
 if __name__ == "__main__":
     test_pd()
+    test_pd_bytes()