mirror of
https://github.com/vale981/hopsflow
synced 2025-03-04 16:31:38 -05:00
lock the dumping of the out file
This commit is contained in:
parent
a5381b3cf9
commit
e8a39d5b03
3 changed files with 52 additions and 4 deletions
|
|
@ -31,6 +31,7 @@ import time
|
|||
import pickle
|
||||
from hops.core.hierarchy_data import HIData
|
||||
from sortedcontainers import SortedList
|
||||
import portalocker
|
||||
|
||||
Aggregate = tuple[int, np.ndarray, np.ndarray]
|
||||
EnsembleReturn = Union[Aggregate, list[Aggregate]]
|
||||
|
|
@ -668,6 +669,8 @@ class WelfordAggregator:
|
|||
return
|
||||
|
||||
self._tracker.add(i)
|
||||
with open("/tmp/out.txt", "a") as f:
|
||||
f.write(f"adding {i}\n")
|
||||
|
||||
self.n += 1
|
||||
delta = new_value - self.mean
|
||||
|
|
@ -784,8 +787,8 @@ def ensemble_mean_online(
|
|||
result = None
|
||||
|
||||
if path.exists():
|
||||
with path.open("rb") as agg_file:
|
||||
aggregate = pickle.load(agg_file)
|
||||
with portalocker.Lock(path, "rb") as agg_file:
|
||||
aggregate: WelfordAggregator = pickle.load(agg_file)
|
||||
if result is not None:
|
||||
aggregate.update(result, i)
|
||||
|
||||
|
|
@ -795,7 +798,7 @@ def ensemble_mean_online(
|
|||
|
||||
aggregate = WelfordAggregator(result, i is not None)
|
||||
|
||||
with path.open("wb") as agg_file:
|
||||
with portalocker.Lock(path, "wb") as agg_file:
|
||||
pickle.dump(aggregate, agg_file)
|
||||
|
||||
return aggregate.ensemble_value
|
||||
|
|
|
|||
46
poetry.lock
generated
46
poetry.lock
generated
|
|
@ -626,6 +626,22 @@ python-versions = ">=3.7"
|
|||
docs = ["furo (>=2022.9.29)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.4)"]
|
||||
test = ["appdirs (==1.4.4)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"]
|
||||
|
||||
[[package]]
|
||||
name = "portalocker"
|
||||
version = "2.6.0"
|
||||
description = "Wraps the portalocker recipe for easy usage"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.5"
|
||||
|
||||
[package.dependencies]
|
||||
pywin32 = {version = ">=226", markers = "platform_system == \"Windows\""}
|
||||
|
||||
[package.extras]
|
||||
docs = ["sphinx (>=1.7.1)"]
|
||||
redis = ["redis"]
|
||||
tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "pytest-timeout (>=2.1.0)", "redis", "sphinx (>=3.0.3)"]
|
||||
|
||||
[[package]]
|
||||
name = "prettytable"
|
||||
version = "3.5.0"
|
||||
|
|
@ -741,6 +757,14 @@ category = "dev"
|
|||
optional = false
|
||||
python-versions = "*"
|
||||
|
||||
[[package]]
|
||||
name = "pywin32"
|
||||
version = "305"
|
||||
description = "Python for Window Extensions"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
|
||||
[[package]]
|
||||
name = "pyyaml"
|
||||
version = "6.0"
|
||||
|
|
@ -1143,7 +1167,7 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools"
|
|||
[metadata]
|
||||
lock-version = "1.1"
|
||||
python-versions = ">=3.9,<3.11"
|
||||
content-hash = "53d4c1d9dfae83142adfad9d1bde9f04ad2443ba9a23075ab355559532ab94af"
|
||||
content-hash = "627b0aae7fea72536c5575cdbec9f518b7dfee6fbae680a08c3357c902af61ea"
|
||||
|
||||
[metadata.files]
|
||||
aiosignal = [
|
||||
|
|
@ -1599,6 +1623,10 @@ platformdirs = [
|
|||
{file = "platformdirs-2.5.4-py3-none-any.whl", hash = "sha256:af0276409f9a02373d540bf8480021a048711d572745aef4b7842dad245eba10"},
|
||||
{file = "platformdirs-2.5.4.tar.gz", hash = "sha256:1006647646d80f16130f052404c6b901e80ee4ed6bef6792e1f238a8969106f7"},
|
||||
]
|
||||
portalocker = [
|
||||
{file = "portalocker-2.6.0-py2.py3-none-any.whl", hash = "sha256:102ed1f2badd8dec9af3d732ef70e94b215b85ba45a8d7ff3c0003f19b442f4e"},
|
||||
{file = "portalocker-2.6.0.tar.gz", hash = "sha256:964f6830fb42a74b5d32bce99ed37d8308c1d7d44ddf18f3dd89f4680de97b39"},
|
||||
]
|
||||
prettytable = [
|
||||
{file = "prettytable-3.5.0-py3-none-any.whl", hash = "sha256:fe391c3b545800028edf5dbb6a5360893feb398367fcc1cf8d7a5b29ce5c59a1"},
|
||||
{file = "prettytable-3.5.0.tar.gz", hash = "sha256:52f682ba4efe29dccb38ff0fe5bac8a23007d0780ff92a8b85af64bc4fc74d72"},
|
||||
|
|
@ -1683,6 +1711,22 @@ pytz = [
|
|||
{file = "pytz-2022.6-py2.py3-none-any.whl", hash = "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427"},
|
||||
{file = "pytz-2022.6.tar.gz", hash = "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2"},
|
||||
]
|
||||
pywin32 = [
|
||||
{file = "pywin32-305-cp310-cp310-win32.whl", hash = "sha256:421f6cd86e84bbb696d54563c48014b12a23ef95a14e0bdba526be756d89f116"},
|
||||
{file = "pywin32-305-cp310-cp310-win_amd64.whl", hash = "sha256:73e819c6bed89f44ff1d690498c0a811948f73777e5f97c494c152b850fad478"},
|
||||
{file = "pywin32-305-cp310-cp310-win_arm64.whl", hash = "sha256:742eb905ce2187133a29365b428e6c3b9001d79accdc30aa8969afba1d8470f4"},
|
||||
{file = "pywin32-305-cp311-cp311-win32.whl", hash = "sha256:19ca459cd2e66c0e2cc9a09d589f71d827f26d47fe4a9d09175f6aa0256b51c2"},
|
||||
{file = "pywin32-305-cp311-cp311-win_amd64.whl", hash = "sha256:326f42ab4cfff56e77e3e595aeaf6c216712bbdd91e464d167c6434b28d65990"},
|
||||
{file = "pywin32-305-cp311-cp311-win_arm64.whl", hash = "sha256:4ecd404b2c6eceaca52f8b2e3e91b2187850a1ad3f8b746d0796a98b4cea04db"},
|
||||
{file = "pywin32-305-cp36-cp36m-win32.whl", hash = "sha256:48d8b1659284f3c17b68587af047d110d8c44837736b8932c034091683e05863"},
|
||||
{file = "pywin32-305-cp36-cp36m-win_amd64.whl", hash = "sha256:13362cc5aa93c2beaf489c9c9017c793722aeb56d3e5166dadd5ef82da021fe1"},
|
||||
{file = "pywin32-305-cp37-cp37m-win32.whl", hash = "sha256:a55db448124d1c1484df22fa8bbcbc45c64da5e6eae74ab095b9ea62e6d00496"},
|
||||
{file = "pywin32-305-cp37-cp37m-win_amd64.whl", hash = "sha256:109f98980bfb27e78f4df8a51a8198e10b0f347257d1e265bb1a32993d0c973d"},
|
||||
{file = "pywin32-305-cp38-cp38-win32.whl", hash = "sha256:9dd98384da775afa009bc04863426cb30596fd78c6f8e4e2e5bbf4edf8029504"},
|
||||
{file = "pywin32-305-cp38-cp38-win_amd64.whl", hash = "sha256:56d7a9c6e1a6835f521788f53b5af7912090674bb84ef5611663ee1595860fc7"},
|
||||
{file = "pywin32-305-cp39-cp39-win32.whl", hash = "sha256:9d968c677ac4d5cbdaa62fd3014ab241718e619d8e36ef8e11fb930515a1e918"},
|
||||
{file = "pywin32-305-cp39-cp39-win_amd64.whl", hash = "sha256:50768c6b7c3f0b38b7fb14dd4104da93ebced5f1a50dc0e834594bff6fbe1271"},
|
||||
]
|
||||
pyyaml = [
|
||||
{file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"},
|
||||
{file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"},
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ ray = "^1.11.0"
|
|||
hops = { git = "git@gitlab.hrz.tu-chemnitz.de:s8896854--tu-dresden.de/hops.git", branch="main" }
|
||||
opt-einsum = "^3.3.0"
|
||||
sortedcontainers = "^2.4.0"
|
||||
portalocker = "^2.6.0"
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
mypy = "^0.910"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue