diff --git a/hopsflow/util.py b/hopsflow/util.py index a11ea46..6acaea6 100644 --- a/hopsflow/util.py +++ b/hopsflow/util.py @@ -31,6 +31,7 @@ import time import pickle from hops.core.hierarchy_data import HIData from sortedcontainers import SortedList +import portalocker Aggregate = tuple[int, np.ndarray, np.ndarray] EnsembleReturn = Union[Aggregate, list[Aggregate]] @@ -668,6 +669,8 @@ class WelfordAggregator: return self._tracker.add(i) + with open("/tmp/out.txt", "a") as f: + f.write(f"adding {i}\n") self.n += 1 delta = new_value - self.mean @@ -784,8 +787,8 @@ def ensemble_mean_online( result = None if path.exists(): - with path.open("rb") as agg_file: - aggregate = pickle.load(agg_file) + with portalocker.Lock(path, "rb") as agg_file: + aggregate: WelfordAggregator = pickle.load(agg_file) if result is not None: aggregate.update(result, i) @@ -795,7 +798,7 @@ def ensemble_mean_online( aggregate = WelfordAggregator(result, i is not None) - with path.open("wb") as agg_file: + with portalocker.Lock(path, "wb") as agg_file: pickle.dump(aggregate, agg_file) return aggregate.ensemble_value diff --git a/poetry.lock b/poetry.lock index a67af38..d1c989c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -626,6 +626,22 @@ python-versions = ">=3.7" docs = ["furo (>=2022.9.29)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.4)"] test = ["appdirs (==1.4.4)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] +[[package]] +name = "portalocker" +version = "2.6.0" +description = "Wraps the portalocker recipe for easy usage" +category = "main" +optional = false +python-versions = ">=3.5" + +[package.dependencies] +pywin32 = {version = ">=226", markers = "platform_system == \"Windows\""} + +[package.extras] +docs = ["sphinx (>=1.7.1)"] +redis = ["redis"] +tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-timeout (>=2.1.0)", "redis", "sphinx (>=3.0.3)"] + [[package]] name = "prettytable" version = "3.5.0" @@ -741,6 +757,14 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "pywin32" +version = "305" +description = "Python for Window Extensions" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "pyyaml" version = "6.0" @@ -1143,7 +1167,7 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [metadata] lock-version = "1.1" python-versions = ">=3.9,<3.11" -content-hash = "53d4c1d9dfae83142adfad9d1bde9f04ad2443ba9a23075ab355559532ab94af" +content-hash = "627b0aae7fea72536c5575cdbec9f518b7dfee6fbae680a08c3357c902af61ea" [metadata.files] aiosignal = [ @@ -1599,6 +1623,10 @@ platformdirs = [ {file = "platformdirs-2.5.4-py3-none-any.whl", hash = "sha256:af0276409f9a02373d540bf8480021a048711d572745aef4b7842dad245eba10"}, {file = "platformdirs-2.5.4.tar.gz", hash = "sha256:1006647646d80f16130f052404c6b901e80ee4ed6bef6792e1f238a8969106f7"}, ] +portalocker = [ + {file = "portalocker-2.6.0-py2.py3-none-any.whl", hash = "sha256:102ed1f2badd8dec9af3d732ef70e94b215b85ba45a8d7ff3c0003f19b442f4e"}, + {file = "portalocker-2.6.0.tar.gz", hash = "sha256:964f6830fb42a74b5d32bce99ed37d8308c1d7d44ddf18f3dd89f4680de97b39"}, +] prettytable = [ {file = "prettytable-3.5.0-py3-none-any.whl", hash = "sha256:fe391c3b545800028edf5dbb6a5360893feb398367fcc1cf8d7a5b29ce5c59a1"}, {file = "prettytable-3.5.0.tar.gz", hash = "sha256:52f682ba4efe29dccb38ff0fe5bac8a23007d0780ff92a8b85af64bc4fc74d72"}, @@ -1683,6 +1711,22 @@ pytz = [ {file = "pytz-2022.6-py2.py3-none-any.whl", hash = "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427"}, {file = "pytz-2022.6.tar.gz", hash = "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2"}, ] +pywin32 = [ + {file = "pywin32-305-cp310-cp310-win32.whl", hash = "sha256:421f6cd86e84bbb696d54563c48014b12a23ef95a14e0bdba526be756d89f116"}, + {file = "pywin32-305-cp310-cp310-win_amd64.whl", hash = "sha256:73e819c6bed89f44ff1d690498c0a811948f73777e5f97c494c152b850fad478"}, + {file = "pywin32-305-cp310-cp310-win_arm64.whl", hash = "sha256:742eb905ce2187133a29365b428e6c3b9001d79accdc30aa8969afba1d8470f4"}, + {file = "pywin32-305-cp311-cp311-win32.whl", hash = "sha256:19ca459cd2e66c0e2cc9a09d589f71d827f26d47fe4a9d09175f6aa0256b51c2"}, + {file = "pywin32-305-cp311-cp311-win_amd64.whl", hash = "sha256:326f42ab4cfff56e77e3e595aeaf6c216712bbdd91e464d167c6434b28d65990"}, + {file = "pywin32-305-cp311-cp311-win_arm64.whl", hash = "sha256:4ecd404b2c6eceaca52f8b2e3e91b2187850a1ad3f8b746d0796a98b4cea04db"}, + {file = "pywin32-305-cp36-cp36m-win32.whl", hash = "sha256:48d8b1659284f3c17b68587af047d110d8c44837736b8932c034091683e05863"}, + {file = "pywin32-305-cp36-cp36m-win_amd64.whl", hash = "sha256:13362cc5aa93c2beaf489c9c9017c793722aeb56d3e5166dadd5ef82da021fe1"}, + {file = "pywin32-305-cp37-cp37m-win32.whl", hash = "sha256:a55db448124d1c1484df22fa8bbcbc45c64da5e6eae74ab095b9ea62e6d00496"}, + {file = "pywin32-305-cp37-cp37m-win_amd64.whl", hash = "sha256:109f98980bfb27e78f4df8a51a8198e10b0f347257d1e265bb1a32993d0c973d"}, + {file = "pywin32-305-cp38-cp38-win32.whl", hash = "sha256:9dd98384da775afa009bc04863426cb30596fd78c6f8e4e2e5bbf4edf8029504"}, + {file = "pywin32-305-cp38-cp38-win_amd64.whl", hash = "sha256:56d7a9c6e1a6835f521788f53b5af7912090674bb84ef5611663ee1595860fc7"}, + {file = "pywin32-305-cp39-cp39-win32.whl", hash = "sha256:9d968c677ac4d5cbdaa62fd3014ab241718e619d8e36ef8e11fb930515a1e918"}, + {file = "pywin32-305-cp39-cp39-win_amd64.whl", hash = "sha256:50768c6b7c3f0b38b7fb14dd4104da93ebced5f1a50dc0e834594bff6fbe1271"}, +] pyyaml = [ {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, diff --git a/pyproject.toml b/pyproject.toml index 5aedb8a..d7e9f1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ ray = "^1.11.0" hops = { git = "git@gitlab.hrz.tu-chemnitz.de:s8896854--tu-dresden.de/hops.git", branch="main" } opt-einsum = "^3.3.0" sortedcontainers = "^2.4.0" +portalocker = "^2.6.0" [tool.poetry.dev-dependencies] mypy = "^0.910"