mirror of
https://github.com/vale981/jobmanager
synced 2025-03-05 18:01:38 -05:00
persistentData seems to work, including some testing
This commit is contained in:
cimatosa
parent
94f2149716
commit
879d89d36d
2 changed files with 236 additions and 27 deletions
|
|
@ -1,8 +1,10 @@
|
|||
import sqlitedict as sqd
|
||||
from os.path import abspath, join, exists
|
||||
import os
|
||||
import traceback
|
||||
|
||||
MAGIC_SIGN = 0xff4a87
|
||||
RESERVED_KEYS = (0, 1)
|
||||
|
||||
class PersistentDataStructure(object):
|
||||
def __init__(self, name, path="./", verbose=0):
|
||||
|
|
@ -20,32 +22,125 @@ class PersistentDataStructure(object):
|
|||
|
||||
# open actual sqltedict
|
||||
self.__filename = join(self.__dir_name, self.__name + '.db')
|
||||
self.db = sqd.SqliteDict(filename = self.__filename, autocommit=True)
|
||||
self.open()
|
||||
|
||||
if not 0 in self.db:
|
||||
self.db[0] = 1
|
||||
if 0 in self.db:
|
||||
self.counter = self.db[0]
|
||||
else:
|
||||
self.counter = 0
|
||||
|
||||
if 1 in self.db:
|
||||
self.sub_data_keys = self.db[1]
|
||||
else:
|
||||
self.sub_data_keys = set()
|
||||
|
||||
def _consistency_check(self):
|
||||
c = 0
|
||||
|
||||
for key in self.db:
|
||||
value = self.db[key]
|
||||
if self.__is_sub_data(value):
|
||||
c += 1
|
||||
assert key in self.sub_data_keys
|
||||
|
||||
assert len(self.sub_data_keys) == c
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
if self.verbose > 1:
|
||||
print("exit called for '{}' in '{}'".format(self.__name, self.__dir_name))
|
||||
print("exit called for {} in {}".format(self.__name, self.__dir_name))
|
||||
self.close()
|
||||
|
||||
def close(self):
|
||||
self.db.close()
|
||||
def open(self):
|
||||
"""
|
||||
open the SQL database at self.__filename = <path>/__<name>/<name>.db
|
||||
as sqlitedict
|
||||
"""
|
||||
if self.verbose > 1:
|
||||
print("closed db '{}' in '{}'".format(self.__name, self.__dir_name))
|
||||
print("open db {} in {}".format(self.__name, self.__dir_name))
|
||||
self.db = sqd.SqliteDict(filename = self.__filename, autocommit=False)
|
||||
|
||||
def is_open(self):
|
||||
"""
|
||||
assume the sqligtedict and therefore the SQL database
|
||||
to be opened when self.db[0] is accessible
|
||||
"""
|
||||
try:
|
||||
self.db[0]
|
||||
return True
|
||||
except:
|
||||
return False
|
||||
|
||||
def is_closed(self):
|
||||
return not self.is_open()
|
||||
|
||||
def close(self):
|
||||
"""
|
||||
close the sqligtedict ans therefore the SQL database
|
||||
"""
|
||||
try:
|
||||
self.db.close()
|
||||
if self.verbose > 1:
|
||||
print("closed db {} in {}".format(self.__name, self.__dir_name))
|
||||
except:
|
||||
if self.verbose > 1:
|
||||
print("db seem already closed {} in {}".format(self.__name, self.__dir_name))
|
||||
|
||||
def erase(self):
|
||||
"""
|
||||
removed the database file from the disk
|
||||
|
||||
this is called recursively for all sub PersistentDataStructure
|
||||
"""
|
||||
if self.verbose > 1:
|
||||
print("erase db {} in {}".format(self.__name, self.__dir_name))
|
||||
|
||||
if self.is_closed():
|
||||
self.open()
|
||||
|
||||
try:
|
||||
|
||||
if self.verbose > 1:
|
||||
print("sub_data_keys:", self.sub_data_keys)
|
||||
for key in self.sub_data_keys:
|
||||
if self.verbose > 1:
|
||||
print("call erase for key:", key, "on file", self.__filename)
|
||||
sub_data = self.getData(key)
|
||||
sub_data.erase()
|
||||
except:
|
||||
traceback.print_exc()
|
||||
finally:
|
||||
self.close()
|
||||
|
||||
os.remove(path = self.__filename)
|
||||
try:
|
||||
os.rmdir(path = self.__dir_name)
|
||||
except OSError as e:
|
||||
if self.verbose > 0:
|
||||
print("Warning: directory structure can not be deleted")
|
||||
print(" {}".format(e))
|
||||
|
||||
def __check_key(self, key):
|
||||
if key == 0:
|
||||
raise RuntimeError("key must not be 0 (reserved key)")
|
||||
"""
|
||||
returns True if the key does NOT collide with some reserved keys
|
||||
|
||||
otherwise a RuntimeError will be raised
|
||||
"""
|
||||
if key in RESERVED_KEYS:
|
||||
raise RuntimeError("key must not be in {} (reserved key)".format(RESERVED_KEYS))
|
||||
|
||||
return True
|
||||
|
||||
def __is_sub_data(self, value):
|
||||
"""
|
||||
determine if the value gotten from the sqlitedict refers
|
||||
to a sub PersistentDataStructure
|
||||
|
||||
this is considered the case if the value itself has an index 'magic'
|
||||
whose value matches a magic sign defined by MAGIC_SIGN
|
||||
"""
|
||||
try:
|
||||
assert value['magic'] == MAGIC_SIGN
|
||||
value.pop('magic')
|
||||
|
|
@ -69,6 +164,7 @@ class PersistentDataStructure(object):
|
|||
|
||||
if overwrite or (not key in self.db):
|
||||
self.db[key] = value
|
||||
self.db.commit()
|
||||
return True
|
||||
|
||||
return False
|
||||
|
|
@ -85,11 +181,20 @@ class PersistentDataStructure(object):
|
|||
managed (simple increasing number)
|
||||
"""
|
||||
if not key in self.db:
|
||||
db0 = self.db[0]
|
||||
new_name = "{}".format(db0)
|
||||
self.db[0] = db0+1
|
||||
self.counter += 1
|
||||
self.sub_data_keys.add(key)
|
||||
if self.verbose > 1:
|
||||
print("new sub_data with key", key)
|
||||
print("sub_data_keys are now", self.sub_data_keys)
|
||||
|
||||
new_name = "{}".format(self.counter)
|
||||
kwargs = {'name': new_name, 'path': self.__dir_name, 'magic': MAGIC_SIGN}
|
||||
|
||||
self.db[0] = self.counter
|
||||
self.db[1] = self.sub_data_keys
|
||||
self.db[key] = kwargs
|
||||
self.db.commit()
|
||||
|
||||
kwargs.pop('magic')
|
||||
return PersistentDataStructure(verbose = self.verbose, **kwargs)
|
||||
else:
|
||||
|
|
@ -97,25 +202,52 @@ class PersistentDataStructure(object):
|
|||
|
||||
def getData(self, key, create_sub_data = False):
|
||||
if key in self.db:
|
||||
if self.verbose > 1:
|
||||
print("getData key exists")
|
||||
value = self.db[key]
|
||||
if self.__is_sub_data(value):
|
||||
return PersistentDataStructure(**value)
|
||||
if self.verbose > 1:
|
||||
print("return subData")
|
||||
return PersistentDataStructure(verbose = self.verbose, **value)
|
||||
else:
|
||||
if self.verbose > 1:
|
||||
print("return normal value")
|
||||
return value
|
||||
else:
|
||||
if not create_sub_data:
|
||||
raise KeyError
|
||||
raise KeyError("key '{}' not found".format(key))
|
||||
else:
|
||||
if self.verbose > 1:
|
||||
print("getData key does NOT exists -> create subData")
|
||||
return self.newSubData(key)
|
||||
|
||||
def __getitem__(self, key):
|
||||
return self.db[key]
|
||||
self.__check_key(key)
|
||||
return self.getData(key, create_sub_data=False)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
self.__check_key(key)
|
||||
if key in self.db:
|
||||
if self.__is_sub_data(self.db[key]):
|
||||
raise RuntimeWarning("values which hold sub_data structures can not be overwritten!")
|
||||
return None
|
||||
|
||||
self.db[key] = value
|
||||
|
||||
if self.verbose > 1:
|
||||
print("set", key, "to", value, "in", self.__filename)
|
||||
self.db[key] = value
|
||||
self.db.commit()
|
||||
|
||||
def __delitem__(self, key):
|
||||
self.__check_key(key)
|
||||
value = self.db[key]
|
||||
if self.__is_sub_data(value):
|
||||
with PersistentDataStructure(verbose = self.verbose, **value) as pds:
|
||||
pds.erase()
|
||||
|
||||
del self.sub_data_keys[key]
|
||||
self.db[1] = sub_data_keys
|
||||
|
||||
del self.db[key]
|
||||
self.db.commit()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,15 +3,16 @@
|
|||
from __future__ import division, print_function
|
||||
|
||||
import sys
|
||||
from os.path import abspath, dirname, split
|
||||
from os import rmdir, remove
|
||||
from os.path import abspath, dirname, split, exists
|
||||
|
||||
# Add parent directory to beginning of path variable
|
||||
sys.path = [split(dirname(abspath(__file__)))[0]] + sys.path
|
||||
|
||||
import jobmanager as jm
|
||||
from jobmanager.persistentData import PersistentDataStructure as PDS
|
||||
|
||||
def test_pd():
|
||||
with jm.persistentData.PersistentDataStructure(name='test_data', verbose=2) as data:
|
||||
with PDS(name='test_data', verbose=0) as data:
|
||||
key = 'a'
|
||||
value = 1
|
||||
data.setData(key=key, value=value)
|
||||
|
|
@ -35,6 +36,59 @@ def test_pd():
|
|||
assert sub_sub_data.getData(key) == 4
|
||||
assert sub_data.getData(key) == 3
|
||||
assert data.getData(key) == 1
|
||||
|
||||
data._consistency_check()
|
||||
|
||||
data.erase()
|
||||
|
||||
def test_reserved_key_catch():
|
||||
with PDS(name='data', verbose=0) as data:
|
||||
# TRY TO READ RESERVED KEYS
|
||||
try:
|
||||
a = data[0]
|
||||
except RuntimeError:
|
||||
pass
|
||||
else:
|
||||
assert False
|
||||
|
||||
try:
|
||||
b = data[1]
|
||||
except RuntimeError:
|
||||
pass
|
||||
else:
|
||||
assert False
|
||||
|
||||
# TRY TO SET RESERVED KEYS
|
||||
try:
|
||||
data[0] = 4
|
||||
except RuntimeError:
|
||||
pass
|
||||
else:
|
||||
assert False
|
||||
|
||||
try:
|
||||
data[1] = 4
|
||||
except RuntimeError:
|
||||
pass
|
||||
else:
|
||||
assert False
|
||||
|
||||
# TRY TO REMOVE RESERVED KEYS
|
||||
try:
|
||||
del data[0]
|
||||
except RuntimeError:
|
||||
pass
|
||||
else:
|
||||
assert False
|
||||
|
||||
try:
|
||||
del data[1]
|
||||
except RuntimeError:
|
||||
pass
|
||||
else:
|
||||
assert False
|
||||
|
||||
data.erase()
|
||||
|
||||
def test_pd_bytes():
|
||||
import pickle
|
||||
|
|
@ -45,23 +99,46 @@ def test_pd_bytes():
|
|||
b1 = pickle.dumps(t1)
|
||||
b2 = pickle.dumps(t2)
|
||||
|
||||
with jm.persistentData.PersistentDataStructure(name='base') as base_data:
|
||||
verbose = 0
|
||||
|
||||
with PDS(name='base', verbose=verbose) as base_data:
|
||||
with base_data.getData(key=b1, create_sub_data=True) as sub_data:
|
||||
for i in range(10):
|
||||
for i in range(2, 10):
|
||||
sub_data[i] = t2
|
||||
|
||||
base_data[b2] = t1
|
||||
|
||||
|
||||
with jm.persistentData.PersistentDataStructure(name='base') as base_data:
|
||||
with base_data.getData(key=b1, create_sub_data=True) as sub_data:
|
||||
for i in range(10):
|
||||
if verbose > 1:
|
||||
print("\nCHECK\n")
|
||||
|
||||
with PDS(name='base', verbose=verbose) as base_data:
|
||||
with base_data.getData(key=b1) as sub_data:
|
||||
for i in range(2, 10):
|
||||
assert sub_data[i] == t2
|
||||
|
||||
assert base_data[b2] == t1
|
||||
|
||||
base_data._consistency_check()
|
||||
|
||||
base_data.erase()
|
||||
|
||||
|
||||
def test_directory_removal():
|
||||
with PDS(name='data', verbose=0) as data:
|
||||
with data.newSubData('s1') as s1:
|
||||
s1['bla'] = 9
|
||||
|
||||
f = open(file=data._PersistentDataStructure__dir_name + '/other_file', mode='w')
|
||||
f.close()
|
||||
|
||||
data.erase()
|
||||
|
||||
assert exists(data._PersistentDataStructure__dir_name)
|
||||
remove(data._PersistentDataStructure__dir_name + '/other_file')
|
||||
rmdir(data._PersistentDataStructure__dir_name)
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_reserved_key_catch()
|
||||
test_pd()
|
||||
test_pd_bytes()
|
||||
test_directory_removal()
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue