Split BaseCollection into BaseStorage and BaseCollection
This commit is contained in:
parent
1453c0b72c
commit
040d8c0fff
@ -49,17 +49,12 @@ def load(configuration):
|
|||||||
else:
|
else:
|
||||||
module = storage_type
|
module = storage_type
|
||||||
try:
|
try:
|
||||||
class_ = import_module(module).Collection
|
class_ = import_module(module).Storage
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise RuntimeError("Failed to load storage module %r: %s" %
|
raise RuntimeError("Failed to load storage module %r: %s" %
|
||||||
(module, e)) from e
|
(module, e)) from e
|
||||||
logger.info("Storage type is %r", storage_type)
|
logger.info("Storage type is %r", storage_type)
|
||||||
|
return class_(configuration)
|
||||||
class CollectionCopy(class_):
|
|
||||||
"""Collection copy, avoids overriding the original class attributes."""
|
|
||||||
CollectionCopy.configuration = configuration
|
|
||||||
CollectionCopy.static_init()
|
|
||||||
return CollectionCopy
|
|
||||||
|
|
||||||
|
|
||||||
class ComponentExistsError(ValueError):
|
class ComponentExistsError(ValueError):
|
||||||
@ -76,17 +71,11 @@ class ComponentNotFoundError(ValueError):
|
|||||||
|
|
||||||
class BaseCollection:
|
class BaseCollection:
|
||||||
|
|
||||||
# Overriden on copy by the "load" function
|
@property
|
||||||
configuration = None
|
def path(self):
|
||||||
|
"""The sanitized path of the collection without leading or
|
||||||
# Properties of instance
|
trailing ``/``."""
|
||||||
"""The sanitized path of the collection without leading or trailing ``/``.
|
raise NotImplementedError
|
||||||
"""
|
|
||||||
path = ""
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def static_init(cls):
|
|
||||||
"""init collection copy"""
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def owner(self):
|
def owner(self):
|
||||||
@ -98,37 +87,6 @@ class BaseCollection:
|
|||||||
"""Collection is a principal."""
|
"""Collection is a principal."""
|
||||||
return bool(self.path) and "/" not in self.path
|
return bool(self.path) and "/" not in self.path
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def discover(cls, path, depth="0"):
|
|
||||||
"""Discover a list of collections under the given ``path``.
|
|
||||||
|
|
||||||
``path`` is sanitized.
|
|
||||||
|
|
||||||
If ``depth`` is "0", only the actual object under ``path`` is
|
|
||||||
returned.
|
|
||||||
|
|
||||||
If ``depth`` is anything but "0", it is considered as "1" and direct
|
|
||||||
children are included in the result.
|
|
||||||
|
|
||||||
The root collection "/" must always exist.
|
|
||||||
|
|
||||||
"""
|
|
||||||
raise NotImplementedError
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def move(cls, item, to_collection, to_href):
|
|
||||||
"""Move an object.
|
|
||||||
|
|
||||||
``item`` is the item to move.
|
|
||||||
|
|
||||||
``to_collection`` is the target collection.
|
|
||||||
|
|
||||||
``to_href`` is the target name in ``to_collection``. An item with the
|
|
||||||
same name might already exist.
|
|
||||||
|
|
||||||
"""
|
|
||||||
raise NotImplementedError
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def etag(self):
|
def etag(self):
|
||||||
"""Encoded as quoted-string (see RFC 2616)."""
|
"""Encoded as quoted-string (see RFC 2616)."""
|
||||||
@ -138,27 +96,6 @@ class BaseCollection:
|
|||||||
etag.update(json.dumps(self.get_meta(), sort_keys=True).encode())
|
etag.update(json.dumps(self.get_meta(), sort_keys=True).encode())
|
||||||
return '"%s"' % etag.hexdigest()
|
return '"%s"' % etag.hexdigest()
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def create_collection(cls, href, items=None, props=None):
|
|
||||||
"""Create a collection.
|
|
||||||
|
|
||||||
``href`` is the sanitized path.
|
|
||||||
|
|
||||||
If the collection already exists and neither ``collection`` nor
|
|
||||||
``props`` are set, this method shouldn't do anything. Otherwise the
|
|
||||||
existing collection must be replaced.
|
|
||||||
|
|
||||||
``collection`` is a list of vobject components.
|
|
||||||
|
|
||||||
``props`` are metadata values for the collection.
|
|
||||||
|
|
||||||
``props["tag"]`` is the type of collection (VCALENDAR or
|
|
||||||
VADDRESSBOOK). If the key ``tag`` is missing, it is guessed from the
|
|
||||||
collection.
|
|
||||||
|
|
||||||
"""
|
|
||||||
raise NotImplementedError
|
|
||||||
|
|
||||||
def sync(self, old_token=None):
|
def sync(self, old_token=None):
|
||||||
"""Get the current sync token and changed items for synchronization.
|
"""Get the current sync token and changed items for synchronization.
|
||||||
|
|
||||||
@ -318,9 +255,69 @@ class BaseCollection:
|
|||||||
return "".join((item.serialize() for item in self.get_all()))
|
return "".join((item.serialize() for item in self.get_all()))
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
@classmethod
|
|
||||||
|
class BaseStorage:
|
||||||
|
def __init__(self, configuration):
|
||||||
|
"""Initialize BaseStorage.
|
||||||
|
|
||||||
|
``configuration`` see ``radicale.config`` module.
|
||||||
|
The ``configuration`` must not change during the lifetime of
|
||||||
|
this object, it is kept as an internal reference.
|
||||||
|
|
||||||
|
"""
|
||||||
|
self.configuration = configuration
|
||||||
|
|
||||||
|
def discover(self, path, depth="0"):
|
||||||
|
"""Discover a list of collections under the given ``path``.
|
||||||
|
|
||||||
|
``path`` is sanitized.
|
||||||
|
|
||||||
|
If ``depth`` is "0", only the actual object under ``path`` is
|
||||||
|
returned.
|
||||||
|
|
||||||
|
If ``depth`` is anything but "0", it is considered as "1" and direct
|
||||||
|
children are included in the result.
|
||||||
|
|
||||||
|
The root collection "/" must always exist.
|
||||||
|
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def move(self, item, to_collection, to_href):
|
||||||
|
"""Move an object.
|
||||||
|
|
||||||
|
``item`` is the item to move.
|
||||||
|
|
||||||
|
``to_collection`` is the target collection.
|
||||||
|
|
||||||
|
``to_href`` is the target name in ``to_collection``. An item with the
|
||||||
|
same name might already exist.
|
||||||
|
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def create_collection(self, href, items=None, props=None):
|
||||||
|
"""Create a collection.
|
||||||
|
|
||||||
|
``href`` is the sanitized path.
|
||||||
|
|
||||||
|
If the collection already exists and neither ``collection`` nor
|
||||||
|
``props`` are set, this method shouldn't do anything. Otherwise the
|
||||||
|
existing collection must be replaced.
|
||||||
|
|
||||||
|
``collection`` is a list of vobject components.
|
||||||
|
|
||||||
|
``props`` are metadata values for the collection.
|
||||||
|
|
||||||
|
``props["tag"]`` is the type of collection (VCALENDAR or
|
||||||
|
VADDRESSBOOK). If the key ``tag`` is missing, it is guessed from the
|
||||||
|
collection.
|
||||||
|
|
||||||
|
"""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
@contextlib.contextmanager
|
@contextlib.contextmanager
|
||||||
def acquire_lock(cls, mode, user=None):
|
def acquire_lock(self, mode, user=None):
|
||||||
"""Set a context manager to lock the whole storage.
|
"""Set a context manager to lock the whole storage.
|
||||||
|
|
||||||
``mode`` must either be "r" for shared access or "w" for exclusive
|
``mode`` must either be "r" for shared access or "w" for exclusive
|
||||||
@ -331,7 +328,6 @@ class BaseCollection:
|
|||||||
"""
|
"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
@classmethod
|
def verify(self):
|
||||||
def verify(cls):
|
|
||||||
"""Check the storage for errors."""
|
"""Check the storage for errors."""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
@ -32,49 +32,40 @@ from tempfile import NamedTemporaryFile
|
|||||||
from radicale import pathutils, storage
|
from radicale import pathutils, storage
|
||||||
from radicale.storage.multifilesystem.cache import CollectionCacheMixin
|
from radicale.storage.multifilesystem.cache import CollectionCacheMixin
|
||||||
from radicale.storage.multifilesystem.create_collection import \
|
from radicale.storage.multifilesystem.create_collection import \
|
||||||
CollectionCreateCollectionMixin
|
StorageCreateCollectionMixin
|
||||||
from radicale.storage.multifilesystem.delete import CollectionDeleteMixin
|
from radicale.storage.multifilesystem.delete import CollectionDeleteMixin
|
||||||
from radicale.storage.multifilesystem.discover import CollectionDiscoverMixin
|
from radicale.storage.multifilesystem.discover import StorageDiscoverMixin
|
||||||
from radicale.storage.multifilesystem.get import CollectionGetMixin
|
from radicale.storage.multifilesystem.get import CollectionGetMixin
|
||||||
from radicale.storage.multifilesystem.history import CollectionHistoryMixin
|
from radicale.storage.multifilesystem.history import CollectionHistoryMixin
|
||||||
from radicale.storage.multifilesystem.lock import CollectionLockMixin
|
from radicale.storage.multifilesystem.lock import (CollectionLockMixin,
|
||||||
|
StorageLockMixin)
|
||||||
from radicale.storage.multifilesystem.meta import CollectionMetaMixin
|
from radicale.storage.multifilesystem.meta import CollectionMetaMixin
|
||||||
from radicale.storage.multifilesystem.move import CollectionMoveMixin
|
from radicale.storage.multifilesystem.move import StorageMoveMixin
|
||||||
from radicale.storage.multifilesystem.sync import CollectionSyncMixin
|
from radicale.storage.multifilesystem.sync import CollectionSyncMixin
|
||||||
from radicale.storage.multifilesystem.upload import CollectionUploadMixin
|
from radicale.storage.multifilesystem.upload import CollectionUploadMixin
|
||||||
from radicale.storage.multifilesystem.verify import CollectionVerifyMixin
|
from radicale.storage.multifilesystem.verify import StorageVerifyMixin
|
||||||
|
|
||||||
|
|
||||||
class Collection(
|
class Collection(
|
||||||
CollectionCacheMixin, CollectionCreateCollectionMixin,
|
CollectionCacheMixin, CollectionDeleteMixin, CollectionGetMixin,
|
||||||
CollectionDeleteMixin, CollectionDiscoverMixin, CollectionGetMixin,
|
|
||||||
CollectionHistoryMixin, CollectionLockMixin, CollectionMetaMixin,
|
CollectionHistoryMixin, CollectionLockMixin, CollectionMetaMixin,
|
||||||
CollectionMoveMixin, CollectionSyncMixin, CollectionUploadMixin,
|
CollectionSyncMixin, CollectionUploadMixin, storage.BaseCollection):
|
||||||
CollectionVerifyMixin, storage.BaseCollection):
|
|
||||||
"""Collection stored in several files per calendar."""
|
|
||||||
|
|
||||||
@classmethod
|
def __init__(self, storage, path, filesystem_path=None):
|
||||||
def static_init(cls):
|
self._storage = storage
|
||||||
folder = cls.configuration.get("storage", "filesystem_folder")
|
folder = storage._get_collection_root_folder()
|
||||||
cls._makedirs_synced(folder)
|
|
||||||
super().static_init()
|
|
||||||
|
|
||||||
def __init__(self, path, filesystem_path=None):
|
|
||||||
folder = self._get_collection_root_folder()
|
|
||||||
# Path should already be sanitized
|
# Path should already be sanitized
|
||||||
self.path = pathutils.strip_path(path)
|
self._path = pathutils.strip_path(path)
|
||||||
self._encoding = self.configuration.get("encoding", "stock")
|
self._encoding = self._storage.configuration.get("encoding", "stock")
|
||||||
if filesystem_path is None:
|
if filesystem_path is None:
|
||||||
filesystem_path = pathutils.path_to_filesystem(folder, self.path)
|
filesystem_path = pathutils.path_to_filesystem(folder, self.path)
|
||||||
self._filesystem_path = filesystem_path
|
self._filesystem_path = filesystem_path
|
||||||
self._etag_cache = None
|
self._etag_cache = None
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
|
||||||
@classmethod
|
@property
|
||||||
def _get_collection_root_folder(cls):
|
def path(self):
|
||||||
filesystem_folder = cls.configuration.get(
|
return self._path
|
||||||
"storage", "filesystem_folder")
|
|
||||||
return os.path.join(filesystem_folder, "collection-root")
|
|
||||||
|
|
||||||
@contextlib.contextmanager
|
@contextlib.contextmanager
|
||||||
def _atomic_write(self, path, mode="w", newline=None, sync_directory=True,
|
def _atomic_write(self, path, mode="w", newline=None, sync_directory=True,
|
||||||
@ -87,7 +78,7 @@ class Collection(
|
|||||||
yield tmp
|
yield tmp
|
||||||
tmp.flush()
|
tmp.flush()
|
||||||
try:
|
try:
|
||||||
self._fsync(tmp.fileno())
|
self._storage._fsync(tmp.fileno())
|
||||||
except OSError as e:
|
except OSError as e:
|
||||||
raise RuntimeError("Fsync'ing file %r failed: %s" %
|
raise RuntimeError("Fsync'ing file %r failed: %s" %
|
||||||
(path, e)) from e
|
(path, e)) from e
|
||||||
@ -98,50 +89,7 @@ class Collection(
|
|||||||
os.remove(tmp.name)
|
os.remove(tmp.name)
|
||||||
raise
|
raise
|
||||||
if sync_directory:
|
if sync_directory:
|
||||||
self._sync_directory(directory)
|
self._storage._sync_directory(directory)
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def _fsync(cls, fd):
|
|
||||||
if cls.configuration.get("internal", "filesystem_fsync"):
|
|
||||||
pathutils.fsync(fd)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def _sync_directory(cls, path):
|
|
||||||
"""Sync directory to disk.
|
|
||||||
|
|
||||||
This only works on POSIX and does nothing on other systems.
|
|
||||||
|
|
||||||
"""
|
|
||||||
if not cls.configuration.get("internal", "filesystem_fsync"):
|
|
||||||
return
|
|
||||||
if os.name == "posix":
|
|
||||||
try:
|
|
||||||
fd = os.open(path, 0)
|
|
||||||
try:
|
|
||||||
cls._fsync(fd)
|
|
||||||
finally:
|
|
||||||
os.close(fd)
|
|
||||||
except OSError as e:
|
|
||||||
raise RuntimeError("Fsync'ing directory %r failed: %s" %
|
|
||||||
(path, e)) from e
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def _makedirs_synced(cls, filesystem_path):
|
|
||||||
"""Recursively create a directory and its parents in a sync'ed way.
|
|
||||||
|
|
||||||
This method acts silently when the folder already exists.
|
|
||||||
|
|
||||||
"""
|
|
||||||
if os.path.isdir(filesystem_path):
|
|
||||||
return
|
|
||||||
parent_filesystem_path = os.path.dirname(filesystem_path)
|
|
||||||
# Prevent infinite loop
|
|
||||||
if filesystem_path != parent_filesystem_path:
|
|
||||||
# Create parent dirs recursively
|
|
||||||
cls._makedirs_synced(parent_filesystem_path)
|
|
||||||
# Possible race!
|
|
||||||
os.makedirs(filesystem_path, exist_ok=True)
|
|
||||||
cls._sync_directory(parent_filesystem_path)
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def last_modified(self):
|
def last_modified(self):
|
||||||
@ -155,6 +103,63 @@ class Collection(
|
|||||||
@property
|
@property
|
||||||
def etag(self):
|
def etag(self):
|
||||||
# reuse cached value if the storage is read-only
|
# reuse cached value if the storage is read-only
|
||||||
if self._lock.locked == "w" or self._etag_cache is None:
|
if self._storage._lock.locked == "w" or self._etag_cache is None:
|
||||||
self._etag_cache = super().etag
|
self._etag_cache = super().etag
|
||||||
return self._etag_cache
|
return self._etag_cache
|
||||||
|
|
||||||
|
|
||||||
|
class Storage(
|
||||||
|
StorageCreateCollectionMixin, StorageDiscoverMixin, StorageLockMixin,
|
||||||
|
StorageMoveMixin, StorageVerifyMixin, storage.BaseStorage):
|
||||||
|
|
||||||
|
_collection_class = Collection
|
||||||
|
|
||||||
|
def __init__(self, configuration):
|
||||||
|
folder = configuration.get("storage", "filesystem_folder")
|
||||||
|
self._makedirs_synced(folder)
|
||||||
|
super().__init__(configuration)
|
||||||
|
|
||||||
|
def _get_collection_root_folder(self):
|
||||||
|
filesystem_folder = self.configuration.get(
|
||||||
|
"storage", "filesystem_folder")
|
||||||
|
return os.path.join(filesystem_folder, "collection-root")
|
||||||
|
|
||||||
|
def _fsync(self, fd):
|
||||||
|
if self.configuration.get("internal", "filesystem_fsync"):
|
||||||
|
pathutils.fsync(fd)
|
||||||
|
|
||||||
|
def _sync_directory(self, path):
|
||||||
|
"""Sync directory to disk.
|
||||||
|
|
||||||
|
This only works on POSIX and does nothing on other systems.
|
||||||
|
|
||||||
|
"""
|
||||||
|
if not self.configuration.get("internal", "filesystem_fsync"):
|
||||||
|
return
|
||||||
|
if os.name == "posix":
|
||||||
|
try:
|
||||||
|
fd = os.open(path, 0)
|
||||||
|
try:
|
||||||
|
self._fsync(fd)
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
except OSError as e:
|
||||||
|
raise RuntimeError("Fsync'ing directory %r failed: %s" %
|
||||||
|
(path, e)) from e
|
||||||
|
|
||||||
|
def _makedirs_synced(self, filesystem_path):
|
||||||
|
"""Recursively create a directory and its parents in a sync'ed way.
|
||||||
|
|
||||||
|
This method acts silently when the folder already exists.
|
||||||
|
|
||||||
|
"""
|
||||||
|
if os.path.isdir(filesystem_path):
|
||||||
|
return
|
||||||
|
parent_filesystem_path = os.path.dirname(filesystem_path)
|
||||||
|
# Prevent infinite loop
|
||||||
|
if filesystem_path != parent_filesystem_path:
|
||||||
|
# Create parent dirs recursively
|
||||||
|
self._makedirs_synced(parent_filesystem_path)
|
||||||
|
# Possible race!
|
||||||
|
os.makedirs(filesystem_path, exist_ok=True)
|
||||||
|
self._sync_directory(parent_filesystem_path)
|
||||||
|
@ -26,8 +26,7 @@ from radicale.log import logger
|
|||||||
|
|
||||||
|
|
||||||
class CollectionCacheMixin:
|
class CollectionCacheMixin:
|
||||||
@classmethod
|
def _clean_cache(self, folder, names, max_age=None):
|
||||||
def _clean_cache(cls, folder, names, max_age=None):
|
|
||||||
"""Delete all ``names`` in ``folder`` that are older than ``max_age``.
|
"""Delete all ``names`` in ``folder`` that are older than ``max_age``.
|
||||||
"""
|
"""
|
||||||
age_limit = time.time() - max_age if max_age is not None else None
|
age_limit = time.time() - max_age if max_age is not None else None
|
||||||
@ -52,7 +51,7 @@ class CollectionCacheMixin:
|
|||||||
continue
|
continue
|
||||||
modified = True
|
modified = True
|
||||||
if modified:
|
if modified:
|
||||||
cls._sync_directory(folder)
|
self._storage._sync_directory(folder)
|
||||||
|
|
||||||
def _item_cache_hash(self, raw_text):
|
def _item_cache_hash(self, raw_text):
|
||||||
_hash = md5()
|
_hash = md5()
|
||||||
@ -71,7 +70,7 @@ class CollectionCacheMixin:
|
|||||||
cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache",
|
cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache",
|
||||||
"item")
|
"item")
|
||||||
content = self._item_cache_content(item, cache_hash)
|
content = self._item_cache_content(item, cache_hash)
|
||||||
self._makedirs_synced(cache_folder)
|
self._storage._makedirs_synced(cache_folder)
|
||||||
try:
|
try:
|
||||||
# Race: Other processes might have created and locked the
|
# Race: Other processes might have created and locked the
|
||||||
# file.
|
# file.
|
||||||
|
@ -22,21 +22,25 @@ from tempfile import TemporaryDirectory
|
|||||||
from radicale import pathutils
|
from radicale import pathutils
|
||||||
|
|
||||||
|
|
||||||
class CollectionCreateCollectionMixin:
|
class StorageCreateCollectionMixin:
|
||||||
@classmethod
|
|
||||||
def create_collection(cls, href, items=None, props=None):
|
def __init__(self, configuration):
|
||||||
folder = cls._get_collection_root_folder()
|
super().__init__(configuration)
|
||||||
|
|
||||||
|
def create_collection(self, href, items=None, props=None):
|
||||||
|
folder = self._get_collection_root_folder()
|
||||||
|
|
||||||
# Path should already be sanitized
|
# Path should already be sanitized
|
||||||
sane_path = pathutils.strip_path(href)
|
sane_path = pathutils.strip_path(href)
|
||||||
filesystem_path = pathutils.path_to_filesystem(folder, sane_path)
|
filesystem_path = pathutils.path_to_filesystem(folder, sane_path)
|
||||||
|
|
||||||
if not props:
|
if not props:
|
||||||
cls._makedirs_synced(filesystem_path)
|
self._makedirs_synced(filesystem_path)
|
||||||
return cls(pathutils.unstrip_path(sane_path, True))
|
return self._collection_class(
|
||||||
|
self, pathutils.unstrip_path(sane_path, True))
|
||||||
|
|
||||||
parent_dir = os.path.dirname(filesystem_path)
|
parent_dir = os.path.dirname(filesystem_path)
|
||||||
cls._makedirs_synced(parent_dir)
|
self._makedirs_synced(parent_dir)
|
||||||
|
|
||||||
# Create a temporary directory with an unsafe name
|
# Create a temporary directory with an unsafe name
|
||||||
with TemporaryDirectory(
|
with TemporaryDirectory(
|
||||||
@ -44,14 +48,15 @@ class CollectionCreateCollectionMixin:
|
|||||||
# The temporary directory itself can't be renamed
|
# The temporary directory itself can't be renamed
|
||||||
tmp_filesystem_path = os.path.join(tmp_dir, "collection")
|
tmp_filesystem_path = os.path.join(tmp_dir, "collection")
|
||||||
os.makedirs(tmp_filesystem_path)
|
os.makedirs(tmp_filesystem_path)
|
||||||
self = cls(pathutils.unstrip_path(sane_path, True),
|
col = self._collection_class(
|
||||||
|
self, pathutils.unstrip_path(sane_path, True),
|
||||||
filesystem_path=tmp_filesystem_path)
|
filesystem_path=tmp_filesystem_path)
|
||||||
self.set_meta(props)
|
col.set_meta(props)
|
||||||
if items is not None:
|
if items is not None:
|
||||||
if props.get("tag") == "VCALENDAR":
|
if props.get("tag") == "VCALENDAR":
|
||||||
self._upload_all_nonatomic(items, suffix=".ics")
|
col._upload_all_nonatomic(items, suffix=".ics")
|
||||||
elif props.get("tag") == "VADDRESSBOOK":
|
elif props.get("tag") == "VADDRESSBOOK":
|
||||||
self._upload_all_nonatomic(items, suffix=".vcf")
|
col._upload_all_nonatomic(items, suffix=".vcf")
|
||||||
|
|
||||||
# This operation is not atomic on the filesystem level but it's
|
# This operation is not atomic on the filesystem level but it's
|
||||||
# very unlikely that one rename operations succeeds while the
|
# very unlikely that one rename operations succeeds while the
|
||||||
@ -59,6 +64,7 @@ class CollectionCreateCollectionMixin:
|
|||||||
if os.path.exists(filesystem_path):
|
if os.path.exists(filesystem_path):
|
||||||
os.rename(filesystem_path, os.path.join(tmp_dir, "delete"))
|
os.rename(filesystem_path, os.path.join(tmp_dir, "delete"))
|
||||||
os.rename(tmp_filesystem_path, filesystem_path)
|
os.rename(tmp_filesystem_path, filesystem_path)
|
||||||
cls._sync_directory(parent_dir)
|
self._sync_directory(parent_dir)
|
||||||
|
|
||||||
return cls(pathutils.unstrip_path(sane_path, True))
|
return self._collection_class(
|
||||||
|
self, pathutils.unstrip_path(sane_path, True))
|
||||||
|
@ -34,9 +34,9 @@ class CollectionDeleteMixin:
|
|||||||
prefix=".Radicale.tmp-", dir=parent_dir) as tmp:
|
prefix=".Radicale.tmp-", dir=parent_dir) as tmp:
|
||||||
os.rename(self._filesystem_path, os.path.join(
|
os.rename(self._filesystem_path, os.path.join(
|
||||||
tmp, os.path.basename(self._filesystem_path)))
|
tmp, os.path.basename(self._filesystem_path)))
|
||||||
self._sync_directory(parent_dir)
|
self._storage._sync_directory(parent_dir)
|
||||||
else:
|
else:
|
||||||
self._sync_directory(parent_dir)
|
self._storage._sync_directory(parent_dir)
|
||||||
else:
|
else:
|
||||||
# Delete an item
|
# Delete an item
|
||||||
if not pathutils.is_safe_filesystem_path_component(href):
|
if not pathutils.is_safe_filesystem_path_component(href):
|
||||||
@ -45,7 +45,7 @@ class CollectionDeleteMixin:
|
|||||||
if not os.path.isfile(path):
|
if not os.path.isfile(path):
|
||||||
raise storage.ComponentNotFoundError(href)
|
raise storage.ComponentNotFoundError(href)
|
||||||
os.remove(path)
|
os.remove(path)
|
||||||
self._sync_directory(os.path.dirname(path))
|
self._storage._sync_directory(os.path.dirname(path))
|
||||||
# Track the change
|
# Track the change
|
||||||
self._update_history_etag(href, None)
|
self._update_history_etag(href, None)
|
||||||
self._clean_history()
|
self._clean_history()
|
||||||
|
@ -24,17 +24,20 @@ from radicale import pathutils
|
|||||||
from radicale.log import logger
|
from radicale.log import logger
|
||||||
|
|
||||||
|
|
||||||
class CollectionDiscoverMixin:
|
class StorageDiscoverMixin:
|
||||||
@classmethod
|
|
||||||
def discover(cls, path, depth="0", child_context_manager=(
|
def __init__(self, configuration):
|
||||||
|
super().__init__(configuration)
|
||||||
|
|
||||||
|
def discover(self, path, depth="0", child_context_manager=(
|
||||||
lambda path, href=None: contextlib.ExitStack())):
|
lambda path, href=None: contextlib.ExitStack())):
|
||||||
# Path should already be sanitized
|
# Path should already be sanitized
|
||||||
sane_path = pathutils.strip_path(path)
|
sane_path = pathutils.strip_path(path)
|
||||||
attributes = sane_path.split("/") if sane_path else []
|
attributes = sane_path.split("/") if sane_path else []
|
||||||
|
|
||||||
folder = cls._get_collection_root_folder()
|
folder = self._get_collection_root_folder()
|
||||||
# Create the root collection
|
# Create the root collection
|
||||||
cls._makedirs_synced(folder)
|
self._makedirs_synced(folder)
|
||||||
try:
|
try:
|
||||||
filesystem_path = pathutils.path_to_filesystem(folder, sane_path)
|
filesystem_path = pathutils.path_to_filesystem(folder, sane_path)
|
||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
@ -53,7 +56,8 @@ class CollectionDiscoverMixin:
|
|||||||
href = None
|
href = None
|
||||||
|
|
||||||
sane_path = "/".join(attributes)
|
sane_path = "/".join(attributes)
|
||||||
collection = cls(pathutils.unstrip_path(sane_path, True))
|
collection = self._collection_class(
|
||||||
|
self, pathutils.unstrip_path(sane_path, True))
|
||||||
|
|
||||||
if href:
|
if href:
|
||||||
yield collection._get(href)
|
yield collection._get(href)
|
||||||
@ -80,4 +84,4 @@ class CollectionDiscoverMixin:
|
|||||||
sane_child_path = posixpath.join(sane_path, href)
|
sane_child_path = posixpath.join(sane_path, href)
|
||||||
child_path = pathutils.unstrip_path(sane_child_path, True)
|
child_path = pathutils.unstrip_path(sane_child_path, True)
|
||||||
with child_context_manager(sane_child_path):
|
with child_context_manager(sane_child_path):
|
||||||
yield cls(child_path)
|
yield self._collection_class(self, child_path)
|
||||||
|
@ -77,7 +77,7 @@ class CollectionGetMixin:
|
|||||||
# Lock the item cache to prevent multpile processes from
|
# Lock the item cache to prevent multpile processes from
|
||||||
# generating the same data in parallel.
|
# generating the same data in parallel.
|
||||||
# This improves the performance for multiple requests.
|
# This improves the performance for multiple requests.
|
||||||
if self._lock.locked == "r":
|
if self._storage._lock.locked == "r":
|
||||||
# Check if another process created the file in the meantime
|
# Check if another process created the file in the meantime
|
||||||
cache_hash, uid, etag, text, name, tag, start, end = \
|
cache_hash, uid, etag, text, name, tag, start, end = \
|
||||||
self._load_item_cache(href, input_hash)
|
self._load_item_cache(href, input_hash)
|
||||||
|
@ -50,7 +50,7 @@ class CollectionHistoryMixin:
|
|||||||
history_etag = binascii.hexlify(os.urandom(16)).decode("ascii")
|
history_etag = binascii.hexlify(os.urandom(16)).decode("ascii")
|
||||||
etag = item.etag if item else ""
|
etag = item.etag if item else ""
|
||||||
if etag != cache_etag:
|
if etag != cache_etag:
|
||||||
self._makedirs_synced(history_folder)
|
self._storage._makedirs_synced(history_folder)
|
||||||
history_etag = radicale_item.get_etag(
|
history_etag = radicale_item.get_etag(
|
||||||
history_etag + "/" + etag).strip("\"")
|
history_etag + "/" + etag).strip("\"")
|
||||||
try:
|
try:
|
||||||
@ -83,5 +83,5 @@ class CollectionHistoryMixin:
|
|||||||
history_folder = os.path.join(self._filesystem_path,
|
history_folder = os.path.join(self._filesystem_path,
|
||||||
".Radicale.cache", "history")
|
".Radicale.cache", "history")
|
||||||
self._clean_cache(history_folder, self._get_deleted_history_hrefs(),
|
self._clean_cache(history_folder, self._get_deleted_history_hrefs(),
|
||||||
max_age=self.configuration.get(
|
max_age=self._storage.configuration.get(
|
||||||
"storage", "max_sync_token_age"))
|
"storage", "max_sync_token_age"))
|
||||||
|
@ -27,32 +27,32 @@ from radicale.log import logger
|
|||||||
|
|
||||||
|
|
||||||
class CollectionLockMixin:
|
class CollectionLockMixin:
|
||||||
@classmethod
|
|
||||||
def static_init(cls):
|
|
||||||
super().static_init()
|
|
||||||
folder = cls.configuration.get("storage", "filesystem_folder")
|
|
||||||
lock_path = os.path.join(folder, ".Radicale.lock")
|
|
||||||
cls._lock = pathutils.RwLock(lock_path)
|
|
||||||
|
|
||||||
def _acquire_cache_lock(self, ns=""):
|
def _acquire_cache_lock(self, ns=""):
|
||||||
if self._lock.locked == "w":
|
if self._storage._lock.locked == "w":
|
||||||
return contextlib.ExitStack()
|
return contextlib.ExitStack()
|
||||||
cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache")
|
cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache")
|
||||||
self._makedirs_synced(cache_folder)
|
self._storage._makedirs_synced(cache_folder)
|
||||||
lock_path = os.path.join(cache_folder,
|
lock_path = os.path.join(cache_folder,
|
||||||
".Radicale.lock" + (".%s" % ns if ns else ""))
|
".Radicale.lock" + (".%s" % ns if ns else ""))
|
||||||
lock = pathutils.RwLock(lock_path)
|
lock = pathutils.RwLock(lock_path)
|
||||||
return lock.acquire("w")
|
return lock.acquire("w")
|
||||||
|
|
||||||
@classmethod
|
|
||||||
|
class StorageLockMixin:
|
||||||
|
def __init__(self, configuration):
|
||||||
|
super().__init__(configuration)
|
||||||
|
folder = self.configuration.get("storage", "filesystem_folder")
|
||||||
|
lock_path = os.path.join(folder, ".Radicale.lock")
|
||||||
|
self._lock = pathutils.RwLock(lock_path)
|
||||||
|
|
||||||
@contextlib.contextmanager
|
@contextlib.contextmanager
|
||||||
def acquire_lock(cls, mode, user=None):
|
def acquire_lock(self, mode, user=None):
|
||||||
with cls._lock.acquire(mode):
|
with self._lock.acquire(mode):
|
||||||
yield
|
yield
|
||||||
# execute hook
|
# execute hook
|
||||||
hook = cls.configuration.get("storage", "hook")
|
hook = self.configuration.get("storage", "hook")
|
||||||
if mode == "w" and hook:
|
if mode == "w" and hook:
|
||||||
folder = cls.configuration.get("storage", "filesystem_folder")
|
folder = self.configuration.get("storage", "filesystem_folder")
|
||||||
logger.debug("Running hook")
|
logger.debug("Running hook")
|
||||||
debug = logger.isEnabledFor(logging.DEBUG)
|
debug = logger.isEnabledFor(logging.DEBUG)
|
||||||
p = subprocess.Popen(
|
p = subprocess.Popen(
|
||||||
|
@ -31,7 +31,7 @@ class CollectionMetaMixin:
|
|||||||
|
|
||||||
def get_meta(self, key=None):
|
def get_meta(self, key=None):
|
||||||
# reuse cached value if the storage is read-only
|
# reuse cached value if the storage is read-only
|
||||||
if self._lock.locked == "w" or self._meta_cache is None:
|
if self._storage._lock.locked == "w" or self._meta_cache is None:
|
||||||
try:
|
try:
|
||||||
try:
|
try:
|
||||||
with open(self._props_path, encoding=self._encoding) as f:
|
with open(self._props_path, encoding=self._encoding) as f:
|
||||||
|
@ -21,9 +21,11 @@ import os
|
|||||||
from radicale import pathutils
|
from radicale import pathutils
|
||||||
|
|
||||||
|
|
||||||
class CollectionMoveMixin:
|
class StorageMoveMixin:
|
||||||
@classmethod
|
def __init__(self, configuration):
|
||||||
def move(cls, item, to_collection, to_href):
|
super().__init__(configuration)
|
||||||
|
|
||||||
|
def move(self, item, to_collection, to_href):
|
||||||
if not pathutils.is_safe_filesystem_path_component(to_href):
|
if not pathutils.is_safe_filesystem_path_component(to_href):
|
||||||
raise pathutils.UnsafePathError(to_href)
|
raise pathutils.UnsafePathError(to_href)
|
||||||
os.replace(
|
os.replace(
|
||||||
@ -31,24 +33,24 @@ class CollectionMoveMixin:
|
|||||||
item.collection._filesystem_path, item.href),
|
item.collection._filesystem_path, item.href),
|
||||||
pathutils.path_to_filesystem(
|
pathutils.path_to_filesystem(
|
||||||
to_collection._filesystem_path, to_href))
|
to_collection._filesystem_path, to_href))
|
||||||
cls._sync_directory(to_collection._filesystem_path)
|
self._sync_directory(to_collection._filesystem_path)
|
||||||
if item.collection._filesystem_path != to_collection._filesystem_path:
|
if item.collection._filesystem_path != to_collection._filesystem_path:
|
||||||
cls._sync_directory(item.collection._filesystem_path)
|
self._sync_directory(item.collection._filesystem_path)
|
||||||
# Move the item cache entry
|
# Move the item cache entry
|
||||||
cache_folder = os.path.join(item.collection._filesystem_path,
|
cache_folder = os.path.join(item.collection._filesystem_path,
|
||||||
".Radicale.cache", "item")
|
".Radicale.cache", "item")
|
||||||
to_cache_folder = os.path.join(to_collection._filesystem_path,
|
to_cache_folder = os.path.join(to_collection._filesystem_path,
|
||||||
".Radicale.cache", "item")
|
".Radicale.cache", "item")
|
||||||
cls._makedirs_synced(to_cache_folder)
|
self._makedirs_synced(to_cache_folder)
|
||||||
try:
|
try:
|
||||||
os.replace(os.path.join(cache_folder, item.href),
|
os.replace(os.path.join(cache_folder, item.href),
|
||||||
os.path.join(to_cache_folder, to_href))
|
os.path.join(to_cache_folder, to_href))
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
pass
|
pass
|
||||||
else:
|
else:
|
||||||
cls._makedirs_synced(to_cache_folder)
|
self._makedirs_synced(to_cache_folder)
|
||||||
if cache_folder != to_cache_folder:
|
if cache_folder != to_cache_folder:
|
||||||
cls._makedirs_synced(cache_folder)
|
self._makedirs_synced(cache_folder)
|
||||||
# Track the change
|
# Track the change
|
||||||
to_collection._update_history_etag(to_href, item)
|
to_collection._update_history_etag(to_href, item)
|
||||||
item.collection._update_history_etag(item.href, None)
|
item.collection._update_history_etag(item.href, None)
|
||||||
|
@ -86,7 +86,7 @@ class CollectionSyncMixin:
|
|||||||
# write the new token state or update the modification time of
|
# write the new token state or update the modification time of
|
||||||
# existing token state
|
# existing token state
|
||||||
if not os.path.exists(token_path):
|
if not os.path.exists(token_path):
|
||||||
self._makedirs_synced(token_folder)
|
self._storage._makedirs_synced(token_folder)
|
||||||
try:
|
try:
|
||||||
# Race: Other processes might have created and locked the file.
|
# Race: Other processes might have created and locked the file.
|
||||||
with self._atomic_write(token_path, "wb") as f:
|
with self._atomic_write(token_path, "wb") as f:
|
||||||
@ -96,7 +96,7 @@ class CollectionSyncMixin:
|
|||||||
else:
|
else:
|
||||||
# clean up old sync tokens and item cache
|
# clean up old sync tokens and item cache
|
||||||
self._clean_cache(token_folder, os.listdir(token_folder),
|
self._clean_cache(token_folder, os.listdir(token_folder),
|
||||||
max_age=self.configuration.get(
|
max_age=self._storage.configuration.get(
|
||||||
"storage", "max_sync_token_age"))
|
"storage", "max_sync_token_age"))
|
||||||
self._clean_history()
|
self._clean_history()
|
||||||
else:
|
else:
|
||||||
|
@ -52,7 +52,7 @@ class CollectionUploadMixin:
|
|||||||
"""
|
"""
|
||||||
cache_folder = os.path.join(self._filesystem_path,
|
cache_folder = os.path.join(self._filesystem_path,
|
||||||
".Radicale.cache", "item")
|
".Radicale.cache", "item")
|
||||||
self._makedirs_synced(cache_folder)
|
self._storage._makedirs_synced(cache_folder)
|
||||||
hrefs = set()
|
hrefs = set()
|
||||||
for item in items:
|
for item in items:
|
||||||
uid = item.uid
|
uid = item.uid
|
||||||
@ -101,5 +101,5 @@ class CollectionUploadMixin:
|
|||||||
with self._atomic_write(os.path.join(cache_folder, href), "wb",
|
with self._atomic_write(os.path.join(cache_folder, href), "wb",
|
||||||
sync_directory=False) as f:
|
sync_directory=False) as f:
|
||||||
pickle.dump(cache_content, f)
|
pickle.dump(cache_content, f)
|
||||||
self._sync_directory(cache_folder)
|
self._storage._sync_directory(cache_folder)
|
||||||
self._sync_directory(self._filesystem_path)
|
self._storage._sync_directory(self._filesystem_path)
|
||||||
|
@ -22,9 +22,8 @@ from radicale import pathutils, storage
|
|||||||
from radicale.log import logger
|
from radicale.log import logger
|
||||||
|
|
||||||
|
|
||||||
class CollectionVerifyMixin:
|
class StorageVerifyMixin:
|
||||||
@classmethod
|
def verify(self):
|
||||||
def verify(cls):
|
|
||||||
item_errors = collection_errors = 0
|
item_errors = collection_errors = 0
|
||||||
|
|
||||||
@contextlib.contextmanager
|
@contextlib.contextmanager
|
||||||
@ -51,7 +50,7 @@ class CollectionVerifyMixin:
|
|||||||
collection = None
|
collection = None
|
||||||
uids = set()
|
uids = set()
|
||||||
has_child_collections = False
|
has_child_collections = False
|
||||||
for item in cls.discover(path, "1", exception_cm):
|
for item in self.discover(path, "1", exception_cm):
|
||||||
if not collection:
|
if not collection:
|
||||||
collection = item
|
collection = item
|
||||||
collection.get_meta()
|
collection.get_meta()
|
||||||
|
@ -25,8 +25,5 @@ Copy of filesystem storage backend for testing
|
|||||||
from radicale.storage import multifilesystem
|
from radicale.storage import multifilesystem
|
||||||
|
|
||||||
|
|
||||||
# TODO: make something more in this collection (and test it)
|
class Storage(multifilesystem.Storage):
|
||||||
class Collection(multifilesystem.Collection):
|
pass
|
||||||
"""Collection stored in a folder."""
|
|
||||||
def __init__(self, *args, **kwargs):
|
|
||||||
super().__init__(*args, **kwargs)
|
|
||||||
|
Loading…
Reference in New Issue
Block a user