From f14e1de071c10b2c2b7803a50ea8779a6b05ce89 Mon Sep 17 00:00:00 2001 From: Unrud Date: Wed, 8 Dec 2021 21:41:12 +0100 Subject: [PATCH] Add multifilesystem_nolock storage --- DOCUMENTATION.md | 3 + config | 2 +- radicale/storage/__init__.py | 2 +- radicale/storage/multifilesystem_nolock.py | 103 +++++++++++++++++++++ radicale/tests/test_base.py | 9 ++ 5 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 radicale/storage/multifilesystem_nolock.py diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 35f9bf3..1de8ebc 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -743,6 +743,9 @@ Available backends: `multifilesystem` : Stores the data in the filesystem. +`multifilesystem_nolock` +: The `multifilesystem` backend without file-based locking. Must only be used with a single process. + Default: `multifilesystem` #### filesystem_folder diff --git a/config b/config index 1d0ee55..7c77f5f 100644 --- a/config +++ b/config @@ -83,7 +83,7 @@ [storage] # Storage backend -# Value: multifilesystem +# Value: multifilesystem | multifilesystem_nolock #type = multifilesystem # Folder for storing local collections, created if not present diff --git a/radicale/storage/__init__.py b/radicale/storage/__init__.py index 0163c7c..c36a070 100644 --- a/radicale/storage/__init__.py +++ b/radicale/storage/__init__.py @@ -37,7 +37,7 @@ from radicale import item as radicale_item from radicale import types, utils from radicale.item import filter as radicale_filter -INTERNAL_TYPES: Sequence[str] = ("multifilesystem",) +INTERNAL_TYPES: Sequence[str] = ("multifilesystem", "multifilesystem_nolock",) CACHE_DEPS: Sequence[str] = ("radicale", "vobject", "python-dateutil",) CACHE_VERSION: bytes = "".join( diff --git a/radicale/storage/multifilesystem_nolock.py b/radicale/storage/multifilesystem_nolock.py new file mode 100644 index 0000000..6bb63a2 --- /dev/null +++ b/radicale/storage/multifilesystem_nolock.py @@ -0,0 +1,103 @@ +# This file is part of Radicale Server - Calendar Server +# Copyright © 2021 Unrud +# +# This library is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Radicale. If not, see . + +""" +The multifilesystem backend without file-based locking. +""" + +import threading +from collections import deque +from typing import Deque, Dict, Iterator, Tuple + +from radicale import config, pathutils, types +from radicale.storage import multifilesystem + + +class RwLock(pathutils.RwLock): + + _cond: threading.Condition + + def __init__(self) -> None: + super().__init__("") + self._cond = threading.Condition(self._lock) + + @types.contextmanager + def acquire(self, mode: str, user: str = "") -> Iterator[None]: + if mode not in "rw": + raise ValueError("Invalid mode: %r" % mode) + with self._cond: + self._cond.wait_for(lambda: not self._writer and ( + mode == "r" or self._readers == 0)) + if mode == "r": + self._readers += 1 + self._cond.notify() + else: + self._writer = True + try: + yield + finally: + with self._cond: + if mode == "r": + self._readers -= 1 + self._writer = False + self._cond.notify() + + +class Collection(multifilesystem.Collection): + + _storage: "Storage" + + @types.contextmanager + def _acquire_cache_lock(self, ns: str = "") -> Iterator[None]: + if self._storage._lock.locked == "w": + yield + return + key = (self.path, ns) + with self._storage._cache_lock: + waiters = self._storage._cache_locks.get(key) + if waiters is None: + self._storage._cache_locks[key] = waiters = deque() + wait = bool(waiters) + waiter = threading.Lock() + waiter.acquire() + waiters.append(waiter) + if wait: + waiter.acquire() + try: + yield + finally: + with self._storage._cache_lock: + removedWaiter = waiters.popleft() + assert removedWaiter is waiter + if waiters: + waiters[0].release() + else: + removedWaiters = self._storage._cache_locks.pop(key) + assert removedWaiters is waiters + + +class Storage(multifilesystem.Storage): + + _collection_class = Collection + + _cache_lock: threading.Lock + _cache_locks: Dict[Tuple[str, str], Deque[threading.Lock]] + + def __init__(self, configuration: config.Configuration) -> None: + super().__init__(configuration) + self._lock = RwLock() + self._cache_lock = threading.Lock() + self._cache_locks = {} diff --git a/radicale/tests/test_base.py b/radicale/tests/test_base.py index 4e8cd11..0c33d53 100644 --- a/radicale/tests/test_base.py +++ b/radicale/tests/test_base.py @@ -1751,6 +1751,15 @@ class TestMultiFileSystem(BaseFileSystemTest, BaseRequestsMixIn): assert "\r\nUID:%s\r\n" % uid in answer +class TestMultiFileSystemNoLock(BaseFileSystemTest): + """Test BaseRequests on multifilesystem_nolock.""" + + storage_type: ClassVar[StorageType] = "multifilesystem_nolock" + + test_add_event = BaseRequestsMixIn.test_add_event + test_item_cache_rebuild = TestMultiFileSystem.test_item_cache_rebuild + + class TestCustomStorageSystem(BaseFileSystemTest): """Test custom backend loading."""