Split BaseCollection into BaseStorage and BaseCollection

This commit is contained in:
Unrud 2020-01-14 06:19:23 +01:00
parent 1453c0b72c
commit 040d8c0fff
15 changed files with 221 additions and 213 deletions

View File

@ -49,17 +49,12 @@ def load(configuration):
else: else:
module = storage_type module = storage_type
try: try:
class_ = import_module(module).Collection class_ = import_module(module).Storage
except Exception as e: except Exception as e:
raise RuntimeError("Failed to load storage module %r: %s" % raise RuntimeError("Failed to load storage module %r: %s" %
(module, e)) from e (module, e)) from e
logger.info("Storage type is %r", storage_type) logger.info("Storage type is %r", storage_type)
return class_(configuration)
class CollectionCopy(class_):
"""Collection copy, avoids overriding the original class attributes."""
CollectionCopy.configuration = configuration
CollectionCopy.static_init()
return CollectionCopy
class ComponentExistsError(ValueError): class ComponentExistsError(ValueError):
@ -76,17 +71,11 @@ class ComponentNotFoundError(ValueError):
class BaseCollection: class BaseCollection:
# Overriden on copy by the "load" function @property
configuration = None def path(self):
"""The sanitized path of the collection without leading or
# Properties of instance trailing ``/``."""
"""The sanitized path of the collection without leading or trailing ``/``. raise NotImplementedError
"""
path = ""
@classmethod
def static_init(cls):
"""init collection copy"""
@property @property
def owner(self): def owner(self):
@ -98,37 +87,6 @@ class BaseCollection:
"""Collection is a principal.""" """Collection is a principal."""
return bool(self.path) and "/" not in self.path return bool(self.path) and "/" not in self.path
@classmethod
def discover(cls, path, depth="0"):
"""Discover a list of collections under the given ``path``.
``path`` is sanitized.
If ``depth`` is "0", only the actual object under ``path`` is
returned.
If ``depth`` is anything but "0", it is considered as "1" and direct
children are included in the result.
The root collection "/" must always exist.
"""
raise NotImplementedError
@classmethod
def move(cls, item, to_collection, to_href):
"""Move an object.
``item`` is the item to move.
``to_collection`` is the target collection.
``to_href`` is the target name in ``to_collection``. An item with the
same name might already exist.
"""
raise NotImplementedError
@property @property
def etag(self): def etag(self):
"""Encoded as quoted-string (see RFC 2616).""" """Encoded as quoted-string (see RFC 2616)."""
@ -138,27 +96,6 @@ class BaseCollection:
etag.update(json.dumps(self.get_meta(), sort_keys=True).encode()) etag.update(json.dumps(self.get_meta(), sort_keys=True).encode())
return '"%s"' % etag.hexdigest() return '"%s"' % etag.hexdigest()
@classmethod
def create_collection(cls, href, items=None, props=None):
"""Create a collection.
``href`` is the sanitized path.
If the collection already exists and neither ``collection`` nor
``props`` are set, this method shouldn't do anything. Otherwise the
existing collection must be replaced.
``collection`` is a list of vobject components.
``props`` are metadata values for the collection.
``props["tag"]`` is the type of collection (VCALENDAR or
VADDRESSBOOK). If the key ``tag`` is missing, it is guessed from the
collection.
"""
raise NotImplementedError
def sync(self, old_token=None): def sync(self, old_token=None):
"""Get the current sync token and changed items for synchronization. """Get the current sync token and changed items for synchronization.
@ -318,9 +255,69 @@ class BaseCollection:
return "".join((item.serialize() for item in self.get_all())) return "".join((item.serialize() for item in self.get_all()))
return "" return ""
@classmethod
class BaseStorage:
def __init__(self, configuration):
"""Initialize BaseStorage.
``configuration`` see ``radicale.config`` module.
The ``configuration`` must not change during the lifetime of
this object, it is kept as an internal reference.
"""
self.configuration = configuration
def discover(self, path, depth="0"):
"""Discover a list of collections under the given ``path``.
``path`` is sanitized.
If ``depth`` is "0", only the actual object under ``path`` is
returned.
If ``depth`` is anything but "0", it is considered as "1" and direct
children are included in the result.
The root collection "/" must always exist.
"""
raise NotImplementedError
def move(self, item, to_collection, to_href):
"""Move an object.
``item`` is the item to move.
``to_collection`` is the target collection.
``to_href`` is the target name in ``to_collection``. An item with the
same name might already exist.
"""
raise NotImplementedError
def create_collection(self, href, items=None, props=None):
"""Create a collection.
``href`` is the sanitized path.
If the collection already exists and neither ``collection`` nor
``props`` are set, this method shouldn't do anything. Otherwise the
existing collection must be replaced.
``collection`` is a list of vobject components.
``props`` are metadata values for the collection.
``props["tag"]`` is the type of collection (VCALENDAR or
VADDRESSBOOK). If the key ``tag`` is missing, it is guessed from the
collection.
"""
raise NotImplementedError
@contextlib.contextmanager @contextlib.contextmanager
def acquire_lock(cls, mode, user=None): def acquire_lock(self, mode, user=None):
"""Set a context manager to lock the whole storage. """Set a context manager to lock the whole storage.
``mode`` must either be "r" for shared access or "w" for exclusive ``mode`` must either be "r" for shared access or "w" for exclusive
@ -331,7 +328,6 @@ class BaseCollection:
""" """
raise NotImplementedError raise NotImplementedError
@classmethod def verify(self):
def verify(cls):
"""Check the storage for errors.""" """Check the storage for errors."""
raise NotImplementedError raise NotImplementedError

View File

@ -32,49 +32,40 @@ from tempfile import NamedTemporaryFile
from radicale import pathutils, storage from radicale import pathutils, storage
from radicale.storage.multifilesystem.cache import CollectionCacheMixin from radicale.storage.multifilesystem.cache import CollectionCacheMixin
from radicale.storage.multifilesystem.create_collection import \ from radicale.storage.multifilesystem.create_collection import \
CollectionCreateCollectionMixin StorageCreateCollectionMixin
from radicale.storage.multifilesystem.delete import CollectionDeleteMixin from radicale.storage.multifilesystem.delete import CollectionDeleteMixin
from radicale.storage.multifilesystem.discover import CollectionDiscoverMixin from radicale.storage.multifilesystem.discover import StorageDiscoverMixin
from radicale.storage.multifilesystem.get import CollectionGetMixin from radicale.storage.multifilesystem.get import CollectionGetMixin
from radicale.storage.multifilesystem.history import CollectionHistoryMixin from radicale.storage.multifilesystem.history import CollectionHistoryMixin
from radicale.storage.multifilesystem.lock import CollectionLockMixin from radicale.storage.multifilesystem.lock import (CollectionLockMixin,
StorageLockMixin)
from radicale.storage.multifilesystem.meta import CollectionMetaMixin from radicale.storage.multifilesystem.meta import CollectionMetaMixin
from radicale.storage.multifilesystem.move import CollectionMoveMixin from radicale.storage.multifilesystem.move import StorageMoveMixin
from radicale.storage.multifilesystem.sync import CollectionSyncMixin from radicale.storage.multifilesystem.sync import CollectionSyncMixin
from radicale.storage.multifilesystem.upload import CollectionUploadMixin from radicale.storage.multifilesystem.upload import CollectionUploadMixin
from radicale.storage.multifilesystem.verify import CollectionVerifyMixin from radicale.storage.multifilesystem.verify import StorageVerifyMixin
class Collection( class Collection(
CollectionCacheMixin, CollectionCreateCollectionMixin, CollectionCacheMixin, CollectionDeleteMixin, CollectionGetMixin,
CollectionDeleteMixin, CollectionDiscoverMixin, CollectionGetMixin,
CollectionHistoryMixin, CollectionLockMixin, CollectionMetaMixin, CollectionHistoryMixin, CollectionLockMixin, CollectionMetaMixin,
CollectionMoveMixin, CollectionSyncMixin, CollectionUploadMixin, CollectionSyncMixin, CollectionUploadMixin, storage.BaseCollection):
CollectionVerifyMixin, storage.BaseCollection):
"""Collection stored in several files per calendar."""
@classmethod def __init__(self, storage, path, filesystem_path=None):
def static_init(cls): self._storage = storage
folder = cls.configuration.get("storage", "filesystem_folder") folder = storage._get_collection_root_folder()
cls._makedirs_synced(folder)
super().static_init()
def __init__(self, path, filesystem_path=None):
folder = self._get_collection_root_folder()
# Path should already be sanitized # Path should already be sanitized
self.path = pathutils.strip_path(path) self._path = pathutils.strip_path(path)
self._encoding = self.configuration.get("encoding", "stock") self._encoding = self._storage.configuration.get("encoding", "stock")
if filesystem_path is None: if filesystem_path is None:
filesystem_path = pathutils.path_to_filesystem(folder, self.path) filesystem_path = pathutils.path_to_filesystem(folder, self.path)
self._filesystem_path = filesystem_path self._filesystem_path = filesystem_path
self._etag_cache = None self._etag_cache = None
super().__init__() super().__init__()
@classmethod @property
def _get_collection_root_folder(cls): def path(self):
filesystem_folder = cls.configuration.get( return self._path
"storage", "filesystem_folder")
return os.path.join(filesystem_folder, "collection-root")
@contextlib.contextmanager @contextlib.contextmanager
def _atomic_write(self, path, mode="w", newline=None, sync_directory=True, def _atomic_write(self, path, mode="w", newline=None, sync_directory=True,
@ -87,7 +78,7 @@ class Collection(
yield tmp yield tmp
tmp.flush() tmp.flush()
try: try:
self._fsync(tmp.fileno()) self._storage._fsync(tmp.fileno())
except OSError as e: except OSError as e:
raise RuntimeError("Fsync'ing file %r failed: %s" % raise RuntimeError("Fsync'ing file %r failed: %s" %
(path, e)) from e (path, e)) from e
@ -98,50 +89,7 @@ class Collection(
os.remove(tmp.name) os.remove(tmp.name)
raise raise
if sync_directory: if sync_directory:
self._sync_directory(directory) self._storage._sync_directory(directory)
@classmethod
def _fsync(cls, fd):
if cls.configuration.get("internal", "filesystem_fsync"):
pathutils.fsync(fd)
@classmethod
def _sync_directory(cls, path):
"""Sync directory to disk.
This only works on POSIX and does nothing on other systems.
"""
if not cls.configuration.get("internal", "filesystem_fsync"):
return
if os.name == "posix":
try:
fd = os.open(path, 0)
try:
cls._fsync(fd)
finally:
os.close(fd)
except OSError as e:
raise RuntimeError("Fsync'ing directory %r failed: %s" %
(path, e)) from e
@classmethod
def _makedirs_synced(cls, filesystem_path):
"""Recursively create a directory and its parents in a sync'ed way.
This method acts silently when the folder already exists.
"""
if os.path.isdir(filesystem_path):
return
parent_filesystem_path = os.path.dirname(filesystem_path)
# Prevent infinite loop
if filesystem_path != parent_filesystem_path:
# Create parent dirs recursively
cls._makedirs_synced(parent_filesystem_path)
# Possible race!
os.makedirs(filesystem_path, exist_ok=True)
cls._sync_directory(parent_filesystem_path)
@property @property
def last_modified(self): def last_modified(self):
@ -155,6 +103,63 @@ class Collection(
@property @property
def etag(self): def etag(self):
# reuse cached value if the storage is read-only # reuse cached value if the storage is read-only
if self._lock.locked == "w" or self._etag_cache is None: if self._storage._lock.locked == "w" or self._etag_cache is None:
self._etag_cache = super().etag self._etag_cache = super().etag
return self._etag_cache return self._etag_cache
class Storage(
StorageCreateCollectionMixin, StorageDiscoverMixin, StorageLockMixin,
StorageMoveMixin, StorageVerifyMixin, storage.BaseStorage):
_collection_class = Collection
def __init__(self, configuration):
folder = configuration.get("storage", "filesystem_folder")
self._makedirs_synced(folder)
super().__init__(configuration)
def _get_collection_root_folder(self):
filesystem_folder = self.configuration.get(
"storage", "filesystem_folder")
return os.path.join(filesystem_folder, "collection-root")
def _fsync(self, fd):
if self.configuration.get("internal", "filesystem_fsync"):
pathutils.fsync(fd)
def _sync_directory(self, path):
"""Sync directory to disk.
This only works on POSIX and does nothing on other systems.
"""
if not self.configuration.get("internal", "filesystem_fsync"):
return
if os.name == "posix":
try:
fd = os.open(path, 0)
try:
self._fsync(fd)
finally:
os.close(fd)
except OSError as e:
raise RuntimeError("Fsync'ing directory %r failed: %s" %
(path, e)) from e
def _makedirs_synced(self, filesystem_path):
"""Recursively create a directory and its parents in a sync'ed way.
This method acts silently when the folder already exists.
"""
if os.path.isdir(filesystem_path):
return
parent_filesystem_path = os.path.dirname(filesystem_path)
# Prevent infinite loop
if filesystem_path != parent_filesystem_path:
# Create parent dirs recursively
self._makedirs_synced(parent_filesystem_path)
# Possible race!
os.makedirs(filesystem_path, exist_ok=True)
self._sync_directory(parent_filesystem_path)

View File

@ -26,8 +26,7 @@ from radicale.log import logger
class CollectionCacheMixin: class CollectionCacheMixin:
@classmethod def _clean_cache(self, folder, names, max_age=None):
def _clean_cache(cls, folder, names, max_age=None):
"""Delete all ``names`` in ``folder`` that are older than ``max_age``. """Delete all ``names`` in ``folder`` that are older than ``max_age``.
""" """
age_limit = time.time() - max_age if max_age is not None else None age_limit = time.time() - max_age if max_age is not None else None
@ -52,7 +51,7 @@ class CollectionCacheMixin:
continue continue
modified = True modified = True
if modified: if modified:
cls._sync_directory(folder) self._storage._sync_directory(folder)
def _item_cache_hash(self, raw_text): def _item_cache_hash(self, raw_text):
_hash = md5() _hash = md5()
@ -71,7 +70,7 @@ class CollectionCacheMixin:
cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache", cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache",
"item") "item")
content = self._item_cache_content(item, cache_hash) content = self._item_cache_content(item, cache_hash)
self._makedirs_synced(cache_folder) self._storage._makedirs_synced(cache_folder)
try: try:
# Race: Other processes might have created and locked the # Race: Other processes might have created and locked the
# file. # file.

View File

@ -22,21 +22,25 @@ from tempfile import TemporaryDirectory
from radicale import pathutils from radicale import pathutils
class CollectionCreateCollectionMixin: class StorageCreateCollectionMixin:
@classmethod
def create_collection(cls, href, items=None, props=None): def __init__(self, configuration):
folder = cls._get_collection_root_folder() super().__init__(configuration)
def create_collection(self, href, items=None, props=None):
folder = self._get_collection_root_folder()
# Path should already be sanitized # Path should already be sanitized
sane_path = pathutils.strip_path(href) sane_path = pathutils.strip_path(href)
filesystem_path = pathutils.path_to_filesystem(folder, sane_path) filesystem_path = pathutils.path_to_filesystem(folder, sane_path)
if not props: if not props:
cls._makedirs_synced(filesystem_path) self._makedirs_synced(filesystem_path)
return cls(pathutils.unstrip_path(sane_path, True)) return self._collection_class(
self, pathutils.unstrip_path(sane_path, True))
parent_dir = os.path.dirname(filesystem_path) parent_dir = os.path.dirname(filesystem_path)
cls._makedirs_synced(parent_dir) self._makedirs_synced(parent_dir)
# Create a temporary directory with an unsafe name # Create a temporary directory with an unsafe name
with TemporaryDirectory( with TemporaryDirectory(
@ -44,14 +48,15 @@ class CollectionCreateCollectionMixin:
# The temporary directory itself can't be renamed # The temporary directory itself can't be renamed
tmp_filesystem_path = os.path.join(tmp_dir, "collection") tmp_filesystem_path = os.path.join(tmp_dir, "collection")
os.makedirs(tmp_filesystem_path) os.makedirs(tmp_filesystem_path)
self = cls(pathutils.unstrip_path(sane_path, True), col = self._collection_class(
self, pathutils.unstrip_path(sane_path, True),
filesystem_path=tmp_filesystem_path) filesystem_path=tmp_filesystem_path)
self.set_meta(props) col.set_meta(props)
if items is not None: if items is not None:
if props.get("tag") == "VCALENDAR": if props.get("tag") == "VCALENDAR":
self._upload_all_nonatomic(items, suffix=".ics") col._upload_all_nonatomic(items, suffix=".ics")
elif props.get("tag") == "VADDRESSBOOK": elif props.get("tag") == "VADDRESSBOOK":
self._upload_all_nonatomic(items, suffix=".vcf") col._upload_all_nonatomic(items, suffix=".vcf")
# This operation is not atomic on the filesystem level but it's # This operation is not atomic on the filesystem level but it's
# very unlikely that one rename operations succeeds while the # very unlikely that one rename operations succeeds while the
@ -59,6 +64,7 @@ class CollectionCreateCollectionMixin:
if os.path.exists(filesystem_path): if os.path.exists(filesystem_path):
os.rename(filesystem_path, os.path.join(tmp_dir, "delete")) os.rename(filesystem_path, os.path.join(tmp_dir, "delete"))
os.rename(tmp_filesystem_path, filesystem_path) os.rename(tmp_filesystem_path, filesystem_path)
cls._sync_directory(parent_dir) self._sync_directory(parent_dir)
return cls(pathutils.unstrip_path(sane_path, True)) return self._collection_class(
self, pathutils.unstrip_path(sane_path, True))

View File

@ -34,9 +34,9 @@ class CollectionDeleteMixin:
prefix=".Radicale.tmp-", dir=parent_dir) as tmp: prefix=".Radicale.tmp-", dir=parent_dir) as tmp:
os.rename(self._filesystem_path, os.path.join( os.rename(self._filesystem_path, os.path.join(
tmp, os.path.basename(self._filesystem_path))) tmp, os.path.basename(self._filesystem_path)))
self._sync_directory(parent_dir) self._storage._sync_directory(parent_dir)
else: else:
self._sync_directory(parent_dir) self._storage._sync_directory(parent_dir)
else: else:
# Delete an item # Delete an item
if not pathutils.is_safe_filesystem_path_component(href): if not pathutils.is_safe_filesystem_path_component(href):
@ -45,7 +45,7 @@ class CollectionDeleteMixin:
if not os.path.isfile(path): if not os.path.isfile(path):
raise storage.ComponentNotFoundError(href) raise storage.ComponentNotFoundError(href)
os.remove(path) os.remove(path)
self._sync_directory(os.path.dirname(path)) self._storage._sync_directory(os.path.dirname(path))
# Track the change # Track the change
self._update_history_etag(href, None) self._update_history_etag(href, None)
self._clean_history() self._clean_history()

View File

@ -24,17 +24,20 @@ from radicale import pathutils
from radicale.log import logger from radicale.log import logger
class CollectionDiscoverMixin: class StorageDiscoverMixin:
@classmethod
def discover(cls, path, depth="0", child_context_manager=( def __init__(self, configuration):
super().__init__(configuration)
def discover(self, path, depth="0", child_context_manager=(
lambda path, href=None: contextlib.ExitStack())): lambda path, href=None: contextlib.ExitStack())):
# Path should already be sanitized # Path should already be sanitized
sane_path = pathutils.strip_path(path) sane_path = pathutils.strip_path(path)
attributes = sane_path.split("/") if sane_path else [] attributes = sane_path.split("/") if sane_path else []
folder = cls._get_collection_root_folder() folder = self._get_collection_root_folder()
# Create the root collection # Create the root collection
cls._makedirs_synced(folder) self._makedirs_synced(folder)
try: try:
filesystem_path = pathutils.path_to_filesystem(folder, sane_path) filesystem_path = pathutils.path_to_filesystem(folder, sane_path)
except ValueError as e: except ValueError as e:
@ -53,7 +56,8 @@ class CollectionDiscoverMixin:
href = None href = None
sane_path = "/".join(attributes) sane_path = "/".join(attributes)
collection = cls(pathutils.unstrip_path(sane_path, True)) collection = self._collection_class(
self, pathutils.unstrip_path(sane_path, True))
if href: if href:
yield collection._get(href) yield collection._get(href)
@ -80,4 +84,4 @@ class CollectionDiscoverMixin:
sane_child_path = posixpath.join(sane_path, href) sane_child_path = posixpath.join(sane_path, href)
child_path = pathutils.unstrip_path(sane_child_path, True) child_path = pathutils.unstrip_path(sane_child_path, True)
with child_context_manager(sane_child_path): with child_context_manager(sane_child_path):
yield cls(child_path) yield self._collection_class(self, child_path)

View File

@ -77,7 +77,7 @@ class CollectionGetMixin:
# Lock the item cache to prevent multpile processes from # Lock the item cache to prevent multpile processes from
# generating the same data in parallel. # generating the same data in parallel.
# This improves the performance for multiple requests. # This improves the performance for multiple requests.
if self._lock.locked == "r": if self._storage._lock.locked == "r":
# Check if another process created the file in the meantime # Check if another process created the file in the meantime
cache_hash, uid, etag, text, name, tag, start, end = \ cache_hash, uid, etag, text, name, tag, start, end = \
self._load_item_cache(href, input_hash) self._load_item_cache(href, input_hash)

View File

@ -50,7 +50,7 @@ class CollectionHistoryMixin:
history_etag = binascii.hexlify(os.urandom(16)).decode("ascii") history_etag = binascii.hexlify(os.urandom(16)).decode("ascii")
etag = item.etag if item else "" etag = item.etag if item else ""
if etag != cache_etag: if etag != cache_etag:
self._makedirs_synced(history_folder) self._storage._makedirs_synced(history_folder)
history_etag = radicale_item.get_etag( history_etag = radicale_item.get_etag(
history_etag + "/" + etag).strip("\"") history_etag + "/" + etag).strip("\"")
try: try:
@ -83,5 +83,5 @@ class CollectionHistoryMixin:
history_folder = os.path.join(self._filesystem_path, history_folder = os.path.join(self._filesystem_path,
".Radicale.cache", "history") ".Radicale.cache", "history")
self._clean_cache(history_folder, self._get_deleted_history_hrefs(), self._clean_cache(history_folder, self._get_deleted_history_hrefs(),
max_age=self.configuration.get( max_age=self._storage.configuration.get(
"storage", "max_sync_token_age")) "storage", "max_sync_token_age"))

View File

@ -27,32 +27,32 @@ from radicale.log import logger
class CollectionLockMixin: class CollectionLockMixin:
@classmethod
def static_init(cls):
super().static_init()
folder = cls.configuration.get("storage", "filesystem_folder")
lock_path = os.path.join(folder, ".Radicale.lock")
cls._lock = pathutils.RwLock(lock_path)
def _acquire_cache_lock(self, ns=""): def _acquire_cache_lock(self, ns=""):
if self._lock.locked == "w": if self._storage._lock.locked == "w":
return contextlib.ExitStack() return contextlib.ExitStack()
cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache") cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache")
self._makedirs_synced(cache_folder) self._storage._makedirs_synced(cache_folder)
lock_path = os.path.join(cache_folder, lock_path = os.path.join(cache_folder,
".Radicale.lock" + (".%s" % ns if ns else "")) ".Radicale.lock" + (".%s" % ns if ns else ""))
lock = pathutils.RwLock(lock_path) lock = pathutils.RwLock(lock_path)
return lock.acquire("w") return lock.acquire("w")
@classmethod
class StorageLockMixin:
def __init__(self, configuration):
super().__init__(configuration)
folder = self.configuration.get("storage", "filesystem_folder")
lock_path = os.path.join(folder, ".Radicale.lock")
self._lock = pathutils.RwLock(lock_path)
@contextlib.contextmanager @contextlib.contextmanager
def acquire_lock(cls, mode, user=None): def acquire_lock(self, mode, user=None):
with cls._lock.acquire(mode): with self._lock.acquire(mode):
yield yield
# execute hook # execute hook
hook = cls.configuration.get("storage", "hook") hook = self.configuration.get("storage", "hook")
if mode == "w" and hook: if mode == "w" and hook:
folder = cls.configuration.get("storage", "filesystem_folder") folder = self.configuration.get("storage", "filesystem_folder")
logger.debug("Running hook") logger.debug("Running hook")
debug = logger.isEnabledFor(logging.DEBUG) debug = logger.isEnabledFor(logging.DEBUG)
p = subprocess.Popen( p = subprocess.Popen(

View File

@ -31,7 +31,7 @@ class CollectionMetaMixin:
def get_meta(self, key=None): def get_meta(self, key=None):
# reuse cached value if the storage is read-only # reuse cached value if the storage is read-only
if self._lock.locked == "w" or self._meta_cache is None: if self._storage._lock.locked == "w" or self._meta_cache is None:
try: try:
try: try:
with open(self._props_path, encoding=self._encoding) as f: with open(self._props_path, encoding=self._encoding) as f:

View File

@ -21,9 +21,11 @@ import os
from radicale import pathutils from radicale import pathutils
class CollectionMoveMixin: class StorageMoveMixin:
@classmethod def __init__(self, configuration):
def move(cls, item, to_collection, to_href): super().__init__(configuration)
def move(self, item, to_collection, to_href):
if not pathutils.is_safe_filesystem_path_component(to_href): if not pathutils.is_safe_filesystem_path_component(to_href):
raise pathutils.UnsafePathError(to_href) raise pathutils.UnsafePathError(to_href)
os.replace( os.replace(
@ -31,24 +33,24 @@ class CollectionMoveMixin:
item.collection._filesystem_path, item.href), item.collection._filesystem_path, item.href),
pathutils.path_to_filesystem( pathutils.path_to_filesystem(
to_collection._filesystem_path, to_href)) to_collection._filesystem_path, to_href))
cls._sync_directory(to_collection._filesystem_path) self._sync_directory(to_collection._filesystem_path)
if item.collection._filesystem_path != to_collection._filesystem_path: if item.collection._filesystem_path != to_collection._filesystem_path:
cls._sync_directory(item.collection._filesystem_path) self._sync_directory(item.collection._filesystem_path)
# Move the item cache entry # Move the item cache entry
cache_folder = os.path.join(item.collection._filesystem_path, cache_folder = os.path.join(item.collection._filesystem_path,
".Radicale.cache", "item") ".Radicale.cache", "item")
to_cache_folder = os.path.join(to_collection._filesystem_path, to_cache_folder = os.path.join(to_collection._filesystem_path,
".Radicale.cache", "item") ".Radicale.cache", "item")
cls._makedirs_synced(to_cache_folder) self._makedirs_synced(to_cache_folder)
try: try:
os.replace(os.path.join(cache_folder, item.href), os.replace(os.path.join(cache_folder, item.href),
os.path.join(to_cache_folder, to_href)) os.path.join(to_cache_folder, to_href))
except FileNotFoundError: except FileNotFoundError:
pass pass
else: else:
cls._makedirs_synced(to_cache_folder) self._makedirs_synced(to_cache_folder)
if cache_folder != to_cache_folder: if cache_folder != to_cache_folder:
cls._makedirs_synced(cache_folder) self._makedirs_synced(cache_folder)
# Track the change # Track the change
to_collection._update_history_etag(to_href, item) to_collection._update_history_etag(to_href, item)
item.collection._update_history_etag(item.href, None) item.collection._update_history_etag(item.href, None)

View File

@ -86,7 +86,7 @@ class CollectionSyncMixin:
# write the new token state or update the modification time of # write the new token state or update the modification time of
# existing token state # existing token state
if not os.path.exists(token_path): if not os.path.exists(token_path):
self._makedirs_synced(token_folder) self._storage._makedirs_synced(token_folder)
try: try:
# Race: Other processes might have created and locked the file. # Race: Other processes might have created and locked the file.
with self._atomic_write(token_path, "wb") as f: with self._atomic_write(token_path, "wb") as f:
@ -96,7 +96,7 @@ class CollectionSyncMixin:
else: else:
# clean up old sync tokens and item cache # clean up old sync tokens and item cache
self._clean_cache(token_folder, os.listdir(token_folder), self._clean_cache(token_folder, os.listdir(token_folder),
max_age=self.configuration.get( max_age=self._storage.configuration.get(
"storage", "max_sync_token_age")) "storage", "max_sync_token_age"))
self._clean_history() self._clean_history()
else: else:

View File

@ -52,7 +52,7 @@ class CollectionUploadMixin:
""" """
cache_folder = os.path.join(self._filesystem_path, cache_folder = os.path.join(self._filesystem_path,
".Radicale.cache", "item") ".Radicale.cache", "item")
self._makedirs_synced(cache_folder) self._storage._makedirs_synced(cache_folder)
hrefs = set() hrefs = set()
for item in items: for item in items:
uid = item.uid uid = item.uid
@ -101,5 +101,5 @@ class CollectionUploadMixin:
with self._atomic_write(os.path.join(cache_folder, href), "wb", with self._atomic_write(os.path.join(cache_folder, href), "wb",
sync_directory=False) as f: sync_directory=False) as f:
pickle.dump(cache_content, f) pickle.dump(cache_content, f)
self._sync_directory(cache_folder) self._storage._sync_directory(cache_folder)
self._sync_directory(self._filesystem_path) self._storage._sync_directory(self._filesystem_path)

View File

@ -22,9 +22,8 @@ from radicale import pathutils, storage
from radicale.log import logger from radicale.log import logger
class CollectionVerifyMixin: class StorageVerifyMixin:
@classmethod def verify(self):
def verify(cls):
item_errors = collection_errors = 0 item_errors = collection_errors = 0
@contextlib.contextmanager @contextlib.contextmanager
@ -51,7 +50,7 @@ class CollectionVerifyMixin:
collection = None collection = None
uids = set() uids = set()
has_child_collections = False has_child_collections = False
for item in cls.discover(path, "1", exception_cm): for item in self.discover(path, "1", exception_cm):
if not collection: if not collection:
collection = item collection = item
collection.get_meta() collection.get_meta()

View File

@ -25,8 +25,5 @@ Copy of filesystem storage backend for testing
from radicale.storage import multifilesystem from radicale.storage import multifilesystem
# TODO: make something more in this collection (and test it) class Storage(multifilesystem.Storage):
class Collection(multifilesystem.Collection): pass
"""Collection stored in a folder."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)