tobias/radicale
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Split BaseCollection into BaseStorage and BaseCollection

Browse Source
This commit is contained in:
Unrud 2020-01-14 06:19:23 +01:00
parent 1453c0b72c
commit 040d8c0fff
15 changed files with 221 additions and 213 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
radicale/storage/__init__.py
View File

@ -49,17 +49,12 @@ def load(configuration):
else:
module = storage_type
try:
class_ = import_module(module).Collection
class_ = import_module(module).Storage
except Exception as e:
raise RuntimeError("Failed to load storage module %r: %s" %
(module, e)) from e
logger.info("Storage type is %r", storage_type)
class CollectionCopy(class_):
"""Collection copy, avoids overriding the original class attributes."""
CollectionCopy.configuration = configuration
CollectionCopy.static_init()
return CollectionCopy
return class_(configuration)
class ComponentExistsError(ValueError):
@ -76,17 +71,11 @@ class ComponentNotFoundError(ValueError):
class BaseCollection:
# Overriden on copy by the "load" function
configuration = None
# Properties of instance
"""The sanitized path of the collection without leading or trailing ``/``.
"""
path = ""
@classmethod
def static_init(cls):
"""init collection copy"""
@property
def path(self):
"""The sanitized path of the collection without leading or
trailing ``/``."""
raise NotImplementedError
@property
def owner(self):
@ -98,37 +87,6 @@ class BaseCollection:
"""Collection is a principal."""
return bool(self.path) and "/" not in self.path
@classmethod
def discover(cls, path, depth="0"):
"""Discover a list of collections under the given ``path``.
``path`` is sanitized.
If ``depth`` is "0", only the actual object under ``path`` is
returned.
If ``depth`` is anything but "0", it is considered as "1" and direct
children are included in the result.
The root collection "/" must always exist.
"""
raise NotImplementedError
@classmethod
def move(cls, item, to_collection, to_href):
"""Move an object.
``item`` is the item to move.
``to_collection`` is the target collection.
``to_href`` is the target name in ``to_collection``. An item with the
same name might already exist.
"""
raise NotImplementedError
@property
def etag(self):
"""Encoded as quoted-string (see RFC 2616)."""
@ -138,27 +96,6 @@ class BaseCollection:
etag.update(json.dumps(self.get_meta(), sort_keys=True).encode())
return '"%s"' % etag.hexdigest()
@classmethod
def create_collection(cls, href, items=None, props=None):
"""Create a collection.
``href`` is the sanitized path.
If the collection already exists and neither ``collection`` nor
``props`` are set, this method shouldn't do anything. Otherwise the
existing collection must be replaced.
``collection`` is a list of vobject components.
``props`` are metadata values for the collection.
``props["tag"]`` is the type of collection (VCALENDAR or
VADDRESSBOOK). If the key ``tag`` is missing, it is guessed from the
collection.
"""
raise NotImplementedError
def sync(self, old_token=None):
"""Get the current sync token and changed items for synchronization.
@ -318,9 +255,69 @@ class BaseCollection:
return "".join((item.serialize() for item in self.get_all()))
return ""
@classmethod
class BaseStorage:
def __init__(self, configuration):
"""Initialize BaseStorage.
``configuration`` see ``radicale.config`` module.
The ``configuration`` must not change during the lifetime of
this object, it is kept as an internal reference.
"""
self.configuration = configuration
def discover(self, path, depth="0"):
"""Discover a list of collections under the given ``path``.
``path`` is sanitized.
If ``depth`` is "0", only the actual object under ``path`` is
returned.
If ``depth`` is anything but "0", it is considered as "1" and direct
children are included in the result.
The root collection "/" must always exist.
"""
raise NotImplementedError
def move(self, item, to_collection, to_href):
"""Move an object.
``item`` is the item to move.
``to_collection`` is the target collection.
``to_href`` is the target name in ``to_collection``. An item with the
same name might already exist.
"""
raise NotImplementedError
def create_collection(self, href, items=None, props=None):
"""Create a collection.
``href`` is the sanitized path.
If the collection already exists and neither ``collection`` nor
``props`` are set, this method shouldn't do anything. Otherwise the
existing collection must be replaced.
``collection`` is a list of vobject components.
``props`` are metadata values for the collection.
``props["tag"]`` is the type of collection (VCALENDAR or
VADDRESSBOOK). If the key ``tag`` is missing, it is guessed from the
collection.
"""
raise NotImplementedError
@contextlib.contextmanager
def acquire_lock(cls, mode, user=None):
def acquire_lock(self, mode, user=None):
"""Set a context manager to lock the whole storage.
``mode`` must either be "r" for shared access or "w" for exclusive
@ -331,7 +328,6 @@ class BaseCollection:
"""
raise NotImplementedError
@classmethod
def verify(cls):
def verify(self):
"""Check the storage for errors."""
raise NotImplementedError

147
radicale/storage/multifilesystem/__init__.py
View File

@ -32,49 +32,40 @@ from tempfile import NamedTemporaryFile
from radicale import pathutils, storage
from radicale.storage.multifilesystem.cache import CollectionCacheMixin
from radicale.storage.multifilesystem.create_collection import \
CollectionCreateCollectionMixin
StorageCreateCollectionMixin
from radicale.storage.multifilesystem.delete import CollectionDeleteMixin
from radicale.storage.multifilesystem.discover import CollectionDiscoverMixin
from radicale.storage.multifilesystem.discover import StorageDiscoverMixin
from radicale.storage.multifilesystem.get import CollectionGetMixin
from radicale.storage.multifilesystem.history import CollectionHistoryMixin
from radicale.storage.multifilesystem.lock import CollectionLockMixin
from radicale.storage.multifilesystem.lock import (CollectionLockMixin,
StorageLockMixin)
from radicale.storage.multifilesystem.meta import CollectionMetaMixin
from radicale.storage.multifilesystem.move import CollectionMoveMixin
from radicale.storage.multifilesystem.move import StorageMoveMixin
from radicale.storage.multifilesystem.sync import CollectionSyncMixin
from radicale.storage.multifilesystem.upload import CollectionUploadMixin
from radicale.storage.multifilesystem.verify import CollectionVerifyMixin
from radicale.storage.multifilesystem.verify import StorageVerifyMixin
class Collection(
CollectionCacheMixin, CollectionCreateCollectionMixin,
CollectionDeleteMixin, CollectionDiscoverMixin, CollectionGetMixin,
CollectionCacheMixin, CollectionDeleteMixin, CollectionGetMixin,
CollectionHistoryMixin, CollectionLockMixin, CollectionMetaMixin,
CollectionMoveMixin, CollectionSyncMixin, CollectionUploadMixin,
CollectionVerifyMixin, storage.BaseCollection):
"""Collection stored in several files per calendar."""
CollectionSyncMixin, CollectionUploadMixin, storage.BaseCollection):
@classmethod
def static_init(cls):
folder = cls.configuration.get("storage", "filesystem_folder")
cls._makedirs_synced(folder)
super().static_init()
def __init__(self, path, filesystem_path=None):
folder = self._get_collection_root_folder()
def __init__(self, storage, path, filesystem_path=None):
self._storage = storage
folder = storage._get_collection_root_folder()
# Path should already be sanitized
self.path = pathutils.strip_path(path)
self._encoding = self.configuration.get("encoding", "stock")
self._path = pathutils.strip_path(path)
self._encoding = self._storage.configuration.get("encoding", "stock")
if filesystem_path is None:
filesystem_path = pathutils.path_to_filesystem(folder, self.path)
self._filesystem_path = filesystem_path
self._etag_cache = None
super().__init__()
@classmethod
def _get_collection_root_folder(cls):
filesystem_folder = cls.configuration.get(
"storage", "filesystem_folder")
return os.path.join(filesystem_folder, "collection-root")
@property
def path(self):
return self._path
@contextlib.contextmanager
def _atomic_write(self, path, mode="w", newline=None, sync_directory=True,
@ -87,7 +78,7 @@ class Collection(
yield tmp
tmp.flush()
try:
self._fsync(tmp.fileno())
self._storage._fsync(tmp.fileno())
except OSError as e:
raise RuntimeError("Fsync'ing file %r failed: %s" %
(path, e)) from e
@ -98,50 +89,7 @@ class Collection(
os.remove(tmp.name)
raise
if sync_directory:
self._sync_directory(directory)
@classmethod
def _fsync(cls, fd):
if cls.configuration.get("internal", "filesystem_fsync"):
pathutils.fsync(fd)
@classmethod
def _sync_directory(cls, path):
"""Sync directory to disk.
This only works on POSIX and does nothing on other systems.
"""
if not cls.configuration.get("internal", "filesystem_fsync"):
return
if os.name == "posix":
try:
fd = os.open(path, 0)
try:
cls._fsync(fd)
finally:
os.close(fd)
except OSError as e:
raise RuntimeError("Fsync'ing directory %r failed: %s" %
(path, e)) from e
@classmethod
def _makedirs_synced(cls, filesystem_path):
"""Recursively create a directory and its parents in a sync'ed way.
This method acts silently when the folder already exists.
"""
if os.path.isdir(filesystem_path):
return
parent_filesystem_path = os.path.dirname(filesystem_path)
# Prevent infinite loop
if filesystem_path != parent_filesystem_path:
# Create parent dirs recursively
cls._makedirs_synced(parent_filesystem_path)
# Possible race!
os.makedirs(filesystem_path, exist_ok=True)
cls._sync_directory(parent_filesystem_path)
self._storage._sync_directory(directory)
@property
def last_modified(self):
@ -155,6 +103,63 @@ class Collection(
@property
def etag(self):
# reuse cached value if the storage is read-only
if self._lock.locked == "w" or self._etag_cache is None:
if self._storage._lock.locked == "w" or self._etag_cache is None:
self._etag_cache = super().etag
return self._etag_cache
class Storage(
StorageCreateCollectionMixin, StorageDiscoverMixin, StorageLockMixin,
StorageMoveMixin, StorageVerifyMixin, storage.BaseStorage):
_collection_class = Collection
def __init__(self, configuration):
folder = configuration.get("storage", "filesystem_folder")
self._makedirs_synced(folder)
super().__init__(configuration)
def _get_collection_root_folder(self):
filesystem_folder = self.configuration.get(
"storage", "filesystem_folder")
return os.path.join(filesystem_folder, "collection-root")
def _fsync(self, fd):
if self.configuration.get("internal", "filesystem_fsync"):
pathutils.fsync(fd)
def _sync_directory(self, path):
"""Sync directory to disk.
This only works on POSIX and does nothing on other systems.
"""
if not self.configuration.get("internal", "filesystem_fsync"):
return
if os.name == "posix":
try:
fd = os.open(path, 0)
try:
self._fsync(fd)
finally:
os.close(fd)
except OSError as e:
raise RuntimeError("Fsync'ing directory %r failed: %s" %
(path, e)) from e
def _makedirs_synced(self, filesystem_path):
"""Recursively create a directory and its parents in a sync'ed way.
This method acts silently when the folder already exists.
"""
if os.path.isdir(filesystem_path):
return
parent_filesystem_path = os.path.dirname(filesystem_path)
# Prevent infinite loop
if filesystem_path != parent_filesystem_path:
# Create parent dirs recursively
self._makedirs_synced(parent_filesystem_path)
# Possible race!
os.makedirs(filesystem_path, exist_ok=True)
self._sync_directory(parent_filesystem_path)

7
radicale/storage/multifilesystem/cache.py
View File

@ -26,8 +26,7 @@ from radicale.log import logger
class CollectionCacheMixin:
@classmethod
def _clean_cache(cls, folder, names, max_age=None):
def _clean_cache(self, folder, names, max_age=None):
"""Delete all ``names`` in ``folder`` that are older than ``max_age``.
"""
age_limit = time.time() - max_age if max_age is not None else None
@ -52,7 +51,7 @@ class CollectionCacheMixin:
continue
modified = True
if modified:
cls._sync_directory(folder)
self._storage._sync_directory(folder)
def _item_cache_hash(self, raw_text):
_hash = md5()
@ -71,7 +70,7 @@ class CollectionCacheMixin:
cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache",
"item")
content = self._item_cache_content(item, cache_hash)
self._makedirs_synced(cache_folder)
self._storage._makedirs_synced(cache_folder)
try:
# Race: Other processes might have created and locked the
# file.

34
radicale/storage/multifilesystem/create_collection.py
View File

@ -22,21 +22,25 @@ from tempfile import TemporaryDirectory
from radicale import pathutils
class CollectionCreateCollectionMixin:
@classmethod
def create_collection(cls, href, items=None, props=None):
folder = cls._get_collection_root_folder()
class StorageCreateCollectionMixin:
def __init__(self, configuration):
super().__init__(configuration)
def create_collection(self, href, items=None, props=None):
folder = self._get_collection_root_folder()
# Path should already be sanitized
sane_path = pathutils.strip_path(href)
filesystem_path = pathutils.path_to_filesystem(folder, sane_path)
if not props:
cls._makedirs_synced(filesystem_path)
return cls(pathutils.unstrip_path(sane_path, True))
self._makedirs_synced(filesystem_path)
return self._collection_class(
self, pathutils.unstrip_path(sane_path, True))
parent_dir = os.path.dirname(filesystem_path)
cls._makedirs_synced(parent_dir)
self._makedirs_synced(parent_dir)
# Create a temporary directory with an unsafe name
with TemporaryDirectory(
@ -44,14 +48,15 @@ class CollectionCreateCollectionMixin:
# The temporary directory itself can't be renamed
tmp_filesystem_path = os.path.join(tmp_dir, "collection")
os.makedirs(tmp_filesystem_path)
self = cls(pathutils.unstrip_path(sane_path, True),
filesystem_path=tmp_filesystem_path)
self.set_meta(props)
col = self._collection_class(
self, pathutils.unstrip_path(sane_path, True),
filesystem_path=tmp_filesystem_path)
col.set_meta(props)
if items is not None:
if props.get("tag") == "VCALENDAR":
self._upload_all_nonatomic(items, suffix=".ics")
col._upload_all_nonatomic(items, suffix=".ics")
elif props.get("tag") == "VADDRESSBOOK":
self._upload_all_nonatomic(items, suffix=".vcf")
col._upload_all_nonatomic(items, suffix=".vcf")
# This operation is not atomic on the filesystem level but it's
# very unlikely that one rename operations succeeds while the
@ -59,6 +64,7 @@ class CollectionCreateCollectionMixin:
if os.path.exists(filesystem_path):
os.rename(filesystem_path, os.path.join(tmp_dir, "delete"))
os.rename(tmp_filesystem_path, filesystem_path)
cls._sync_directory(parent_dir)
self._sync_directory(parent_dir)
return cls(pathutils.unstrip_path(sane_path, True))
return self._collection_class(
self, pathutils.unstrip_path(sane_path, True))

6
radicale/storage/multifilesystem/delete.py
View File

@ -34,9 +34,9 @@ class CollectionDeleteMixin:
prefix=".Radicale.tmp-", dir=parent_dir) as tmp:
os.rename(self._filesystem_path, os.path.join(
tmp, os.path.basename(self._filesystem_path)))
self._sync_directory(parent_dir)
self._storage._sync_directory(parent_dir)
else:
self._sync_directory(parent_dir)
self._storage._sync_directory(parent_dir)
else:
# Delete an item
if not pathutils.is_safe_filesystem_path_component(href):
@ -45,7 +45,7 @@ class CollectionDeleteMixin:
if not os.path.isfile(path):
raise storage.ComponentNotFoundError(href)
os.remove(path)
self._sync_directory(os.path.dirname(path))
self._storage._sync_directory(os.path.dirname(path))
# Track the change
self._update_history_etag(href, None)
self._clean_history()

18
radicale/storage/multifilesystem/discover.py
View File

@ -24,17 +24,20 @@ from radicale import pathutils
from radicale.log import logger
class CollectionDiscoverMixin:
@classmethod
def discover(cls, path, depth="0", child_context_manager=(
class StorageDiscoverMixin:
def __init__(self, configuration):
super().__init__(configuration)
def discover(self, path, depth="0", child_context_manager=(
lambda path, href=None: contextlib.ExitStack())):
# Path should already be sanitized
sane_path = pathutils.strip_path(path)
attributes = sane_path.split("/") if sane_path else []
folder = cls._get_collection_root_folder()
folder = self._get_collection_root_folder()
# Create the root collection
cls._makedirs_synced(folder)
self._makedirs_synced(folder)
try:
filesystem_path = pathutils.path_to_filesystem(folder, sane_path)
except ValueError as e:
@ -53,7 +56,8 @@ class CollectionDiscoverMixin:
href = None
sane_path = "/".join(attributes)
collection = cls(pathutils.unstrip_path(sane_path, True))
collection = self._collection_class(
self, pathutils.unstrip_path(sane_path, True))
if href:
yield collection._get(href)
@ -80,4 +84,4 @@ class CollectionDiscoverMixin:
sane_child_path = posixpath.join(sane_path, href)
child_path = pathutils.unstrip_path(sane_child_path, True)
with child_context_manager(sane_child_path):
yield cls(child_path)
yield self._collection_class(self, child_path)

2
radicale/storage/multifilesystem/get.py
View File

@ -77,7 +77,7 @@ class CollectionGetMixin:
# Lock the item cache to prevent multpile processes from
# generating the same data in parallel.
# This improves the performance for multiple requests.
if self._lock.locked == "r":
if self._storage._lock.locked == "r":
# Check if another process created the file in the meantime
cache_hash, uid, etag, text, name, tag, start, end = \
self._load_item_cache(href, input_hash)

4
radicale/storage/multifilesystem/history.py
View File

@ -50,7 +50,7 @@ class CollectionHistoryMixin:
history_etag = binascii.hexlify(os.urandom(16)).decode("ascii")
etag = item.etag if item else ""
if etag != cache_etag:
self._makedirs_synced(history_folder)
self._storage._makedirs_synced(history_folder)
history_etag = radicale_item.get_etag(
history_etag + "/" + etag).strip("\"")
try:
@ -83,5 +83,5 @@ class CollectionHistoryMixin:
history_folder = os.path.join(self._filesystem_path,
".Radicale.cache", "history")
self._clean_cache(history_folder, self._get_deleted_history_hrefs(),
max_age=self.configuration.get(
max_age=self._storage.configuration.get(
"storage", "max_sync_token_age"))

28
radicale/storage/multifilesystem/lock.py
View File

@ -27,32 +27,32 @@ from radicale.log import logger
class CollectionLockMixin:
@classmethod
def static_init(cls):
super().static_init()
folder = cls.configuration.get("storage", "filesystem_folder")
lock_path = os.path.join(folder, ".Radicale.lock")
cls._lock = pathutils.RwLock(lock_path)
def _acquire_cache_lock(self, ns=""):
if self._lock.locked == "w":
if self._storage._lock.locked == "w":
return contextlib.ExitStack()
cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache")
self._makedirs_synced(cache_folder)
self._storage._makedirs_synced(cache_folder)
lock_path = os.path.join(cache_folder,
".Radicale.lock" + (".%s" % ns if ns else ""))
lock = pathutils.RwLock(lock_path)
return lock.acquire("w")
@classmethod
class StorageLockMixin:
def __init__(self, configuration):
super().__init__(configuration)
folder = self.configuration.get("storage", "filesystem_folder")
lock_path = os.path.join(folder, ".Radicale.lock")
self._lock = pathutils.RwLock(lock_path)
@contextlib.contextmanager
def acquire_lock(cls, mode, user=None):
with cls._lock.acquire(mode):
def acquire_lock(self, mode, user=None):
with self._lock.acquire(mode):
yield
# execute hook
hook = cls.configuration.get("storage", "hook")
hook = self.configuration.get("storage", "hook")
if mode == "w" and hook:
folder = cls.configuration.get("storage", "filesystem_folder")
folder = self.configuration.get("storage", "filesystem_folder")
logger.debug("Running hook")
debug = logger.isEnabledFor(logging.DEBUG)
p = subprocess.Popen(

2
radicale/storage/multifilesystem/meta.py
View File

@ -31,7 +31,7 @@ class CollectionMetaMixin:
def get_meta(self, key=None):
# reuse cached value if the storage is read-only
if self._lock.locked == "w" or self._meta_cache is None:
if self._storage._lock.locked == "w" or self._meta_cache is None:
try:
try:
with open(self._props_path, encoding=self._encoding) as f:

18
radicale/storage/multifilesystem/move.py
View File

@ -21,9 +21,11 @@ import os
from radicale import pathutils
class CollectionMoveMixin:
@classmethod
def move(cls, item, to_collection, to_href):
class StorageMoveMixin:
def __init__(self, configuration):
super().__init__(configuration)
def move(self, item, to_collection, to_href):
if not pathutils.is_safe_filesystem_path_component(to_href):
raise pathutils.UnsafePathError(to_href)
os.replace(
@ -31,24 +33,24 @@ class CollectionMoveMixin:
item.collection._filesystem_path, item.href),
pathutils.path_to_filesystem(
to_collection._filesystem_path, to_href))
cls._sync_directory(to_collection._filesystem_path)
self._sync_directory(to_collection._filesystem_path)
if item.collection._filesystem_path != to_collection._filesystem_path:
cls._sync_directory(item.collection._filesystem_path)
self._sync_directory(item.collection._filesystem_path)
# Move the item cache entry
cache_folder = os.path.join(item.collection._filesystem_path,
".Radicale.cache", "item")
to_cache_folder = os.path.join(to_collection._filesystem_path,
".Radicale.cache", "item")
cls._makedirs_synced(to_cache_folder)
self._makedirs_synced(to_cache_folder)
try:
os.replace(os.path.join(cache_folder, item.href),
os.path.join(to_cache_folder, to_href))
except FileNotFoundError:
pass
else:
cls._makedirs_synced(to_cache_folder)
self._makedirs_synced(to_cache_folder)
if cache_folder != to_cache_folder:
cls._makedirs_synced(cache_folder)
self._makedirs_synced(cache_folder)
# Track the change
to_collection._update_history_etag(to_href, item)
item.collection._update_history_etag(item.href, None)

4
radicale/storage/multifilesystem/sync.py
View File

@ -86,7 +86,7 @@ class CollectionSyncMixin:
# write the new token state or update the modification time of
# existing token state
if not os.path.exists(token_path):
self._makedirs_synced(token_folder)
self._storage._makedirs_synced(token_folder)
try:
# Race: Other processes might have created and locked the file.
with self._atomic_write(token_path, "wb") as f:
@ -96,7 +96,7 @@ class CollectionSyncMixin:
else:
# clean up old sync tokens and item cache
self._clean_cache(token_folder, os.listdir(token_folder),
max_age=self.configuration.get(
max_age=self._storage.configuration.get(
"storage", "max_sync_token_age"))
self._clean_history()
else:

6
radicale/storage/multifilesystem/upload.py
View File

@ -52,7 +52,7 @@ class CollectionUploadMixin:
"""
cache_folder = os.path.join(self._filesystem_path,
".Radicale.cache", "item")
self._makedirs_synced(cache_folder)
self._storage._makedirs_synced(cache_folder)
hrefs = set()
for item in items:
uid = item.uid
@ -101,5 +101,5 @@ class CollectionUploadMixin:
with self._atomic_write(os.path.join(cache_folder, href), "wb",
sync_directory=False) as f:
pickle.dump(cache_content, f)
self._sync_directory(cache_folder)
self._sync_directory(self._filesystem_path)
self._storage._sync_directory(cache_folder)
self._storage._sync_directory(self._filesystem_path)

7
radicale/storage/multifilesystem/verify.py
View File

@ -22,9 +22,8 @@ from radicale import pathutils, storage
from radicale.log import logger
class CollectionVerifyMixin:
@classmethod
def verify(cls):
class StorageVerifyMixin:
def verify(self):
item_errors = collection_errors = 0
@contextlib.contextmanager
@ -51,7 +50,7 @@ class CollectionVerifyMixin:
collection = None
uids = set()
has_child_collections = False
for item in cls.discover(path, "1", exception_cm):
for item in self.discover(path, "1", exception_cm):
if not collection:
collection = item
collection.get_meta()

7
radicale/tests/custom/storage.py
View File

@ -25,8 +25,5 @@ Copy of filesystem storage backend for testing
from radicale.storage import multifilesystem
# TODO: make something more in this collection (and test it)
class Collection(multifilesystem.Collection):
"""Collection stored in a folder."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
class Storage(multifilesystem.Storage):
pass