Merge branch 'storage' of https://github.com/Unrud/Radicale into Unrud-storage
This commit is contained in:
Guillaume Ayoub
commit
04764c2af4
@ -100,7 +100,11 @@ def load(configuration, logger):
|
||||
|
||||
|
||||
def get_etag(text):
|
||||
"""Etag from collection or item."""
|
||||
"""Etag from collection or item.
|
||||
|
||||
Encoded as quoted-string (see RFC 2616).
|
||||
|
||||
"""
|
||||
etag = md5()
|
||||
etag.update(text.encode("utf-8"))
|
||||
return '"%s"' % etag.hexdigest()
|
||||
@ -121,7 +125,7 @@ def sanitize_path(path):
|
||||
path = posixpath.normpath(path)
|
||||
new_path = "/"
|
||||
for part in path.split("/"):
|
||||
if not part or part in (".", ".."):
|
||||
if not is_safe_path_component(part):
|
||||
continue
|
||||
new_path = posixpath.join(new_path, part)
|
||||
trailing_slash = "" if new_path.endswith("/") else trailing_slash
|
||||
@ -138,7 +142,8 @@ def is_safe_path_component(path):
|
||||
|
||||
|
||||
def is_safe_filesystem_path_component(path):
|
||||
"""Check if path is a single component of a filesystem path.
|
||||
"""Check if path is a single component of a local and posix filesystem
|
||||
path.
|
||||
|
||||
Check that the path is safe to join too.
|
||||
|
||||
@ -146,7 +151,8 @@ def is_safe_filesystem_path_component(path):
|
||||
return (
|
||||
path and not os.path.splitdrive(path)[0] and
|
||||
not os.path.split(path)[0] and path not in (os.curdir, os.pardir) and
|
||||
not path.startswith(".") and not path.endswith("~"))
|
||||
not path.startswith(".") and not path.endswith("~") and
|
||||
is_safe_path_component(path))
|
||||
|
||||
|
||||
def path_to_filesystem(root, *paths):
|
||||
@ -187,12 +193,6 @@ class ComponentNotFoundError(ValueError):
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
class EtagMismatchError(ValueError):
|
||||
def __init__(self, etag1, etag2):
|
||||
message = "ETags don't match: %s != %s" % (etag1, etag2)
|
||||
super().__init__(message)
|
||||
|
||||
|
||||
class Item:
|
||||
def __init__(self, collection, item, href, last_modified=None):
|
||||
self.collection = collection
|
||||
@ -205,6 +205,7 @@ class Item:
|
||||
|
||||
@property
|
||||
def etag(self):
|
||||
"""Encoded as quoted-string (see RFC 2616)."""
|
||||
return get_etag(self.serialize())
|
||||
|
||||
|
||||
@ -259,6 +260,7 @@ class BaseCollection:
|
||||
|
||||
@property
|
||||
def etag(self):
|
||||
"""Encoded as quoted-string (see RFC 2616)."""
|
||||
return get_etag(self.serialize())
|
||||
|
||||
@classmethod
|
||||
@ -389,11 +391,7 @@ class Collection(BaseCollection):
|
||||
delete=False, prefix=".Radicale.tmp-", newline=newline)
|
||||
try:
|
||||
yield tmp
|
||||
if self.configuration.getboolean("storage", "filesystem_fsync"):
|
||||
if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"):
|
||||
fcntl.fcntl(tmp.fileno(), fcntl.F_FULLFSYNC)
|
||||
else:
|
||||
os.fsync(tmp.fileno())
|
||||
self._fsync(tmp.fileno())
|
||||
tmp.close()
|
||||
os.replace(tmp.name, path)
|
||||
except:
|
||||
@ -411,6 +409,14 @@ class Collection(BaseCollection):
|
||||
return file_name
|
||||
raise FileExistsError(errno.EEXIST, "No usable file name found")
|
||||
|
||||
@classmethod
|
||||
def _fsync(cls, fd):
|
||||
if cls.configuration.getboolean("storage", "filesystem_fsync"):
|
||||
if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"):
|
||||
fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
|
||||
else:
|
||||
os.fsync(fd)
|
||||
|
||||
@classmethod
|
||||
def _sync_directory(cls, path):
|
||||
"""Sync directory to disk.
|
||||
@ -423,10 +429,7 @@ class Collection(BaseCollection):
|
||||
if os.name == "posix":
|
||||
fd = os.open(path, 0)
|
||||
try:
|
||||
if hasattr(fcntl, "F_FULLFSYNC"):
|
||||
fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
|
||||
else:
|
||||
os.fsync(fd)
|
||||
cls._fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
@ -581,23 +584,21 @@ class Collection(BaseCollection):
|
||||
"""
|
||||
fs = []
|
||||
for href, item in vobject_items.items():
|
||||
if not is_safe_filesystem_path_component(href):
|
||||
raise UnsafePathError(href)
|
||||
path = path_to_filesystem(self._filesystem_path, href)
|
||||
fs.append(open(path, "w", encoding=self.encoding, newline=""))
|
||||
fs[-1].write(item.serialize())
|
||||
fsync_fn = lambda fd: None
|
||||
if self.configuration.getboolean("storage", "filesystem_fsync"):
|
||||
if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"):
|
||||
fsync_fn = lambda fd: fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
|
||||
else:
|
||||
fsync_fn = os.fsync
|
||||
# sync everything at once because it's slightly faster.
|
||||
for f in fs:
|
||||
fsync_fn(f.fileno())
|
||||
self._fsync(f.fileno())
|
||||
f.close()
|
||||
self._sync_directory(self._filesystem_path)
|
||||
|
||||
@classmethod
|
||||
def move(cls, item, to_collection, to_href):
|
||||
if not is_safe_filesystem_path_component(to_href):
|
||||
raise UnsafePathError(to_href)
|
||||
os.replace(
|
||||
path_to_filesystem(item.collection._filesystem_path, item.href),
|
||||
path_to_filesystem(to_collection._filesystem_path, to_href))
|
||||
@ -606,12 +607,7 @@ class Collection(BaseCollection):
|
||||
cls._sync_directory(item.collection._filesystem_path)
|
||||
|
||||
def list(self):
|
||||
try:
|
||||
hrefs = os.listdir(self._filesystem_path)
|
||||
except IOError:
|
||||
return
|
||||
|
||||
for href in hrefs:
|
||||
for href in os.listdir(self._filesystem_path):
|
||||
if not is_safe_filesystem_path_component(href):
|
||||
if not href.startswith(".Radicale"):
|
||||
self.logger.debug("Skipping component: %s", href)
|
||||
@ -623,7 +619,6 @@ class Collection(BaseCollection):
|
||||
def get(self, href):
|
||||
if not href:
|
||||
return None
|
||||
href = href.strip("{}").replace("/", "_")
|
||||
if not is_safe_filesystem_path_component(href):
|
||||
self.logger.debug(
|
||||
"Can't translate name safely to filesystem: %s", href)
|
||||
@ -631,15 +626,17 @@ class Collection(BaseCollection):
|
||||
path = path_to_filesystem(self._filesystem_path, href)
|
||||
if not os.path.isfile(path):
|
||||
return None
|
||||
with open(path, encoding=self.encoding, newline="") as fd:
|
||||
text = fd.read()
|
||||
with open(path, encoding=self.encoding, newline="") as f:
|
||||
text = f.read()
|
||||
last_modified = time.strftime(
|
||||
"%a, %d %b %Y %H:%M:%S GMT",
|
||||
time.gmtime(os.path.getmtime(path)))
|
||||
return Item(self, vobject.readOne(text), href, last_modified)
|
||||
|
||||
def has(self, href):
|
||||
return self.get(href) is not None
|
||||
try:
|
||||
item = vobject.readOne(text)
|
||||
except Exception:
|
||||
self.logger.error("Failed to parse component: %s", href)
|
||||
raise
|
||||
return Item(self, item, href, last_modified)
|
||||
|
||||
def upload(self, href, vobject_item):
|
||||
if not is_safe_filesystem_path_component(href):
|
||||
@ -653,18 +650,17 @@ class Collection(BaseCollection):
|
||||
def delete(self, href=None):
|
||||
if href is None:
|
||||
# Delete the collection
|
||||
if os.path.isdir(self._filesystem_path):
|
||||
parent_dir = os.path.dirname(self._filesystem_path)
|
||||
try:
|
||||
os.rmdir(self._filesystem_path)
|
||||
except OSError:
|
||||
with TemporaryDirectory(
|
||||
prefix=".Radicale.tmp-", dir=parent_dir) as tmp:
|
||||
os.rename(self._filesystem_path, os.path.join(
|
||||
tmp, os.path.basename(self._filesystem_path)))
|
||||
self._sync_directory(parent_dir)
|
||||
else:
|
||||
parent_dir = os.path.dirname(self._filesystem_path)
|
||||
try:
|
||||
os.rmdir(self._filesystem_path)
|
||||
except OSError:
|
||||
with TemporaryDirectory(
|
||||
prefix=".Radicale.tmp-", dir=parent_dir) as tmp:
|
||||
os.rename(self._filesystem_path, os.path.join(
|
||||
tmp, os.path.basename(self._filesystem_path)))
|
||||
self._sync_directory(parent_dir)
|
||||
else:
|
||||
self._sync_directory(parent_dir)
|
||||
else:
|
||||
# Delete an item
|
||||
if not is_safe_filesystem_path_component(href):
|
||||
@ -677,40 +673,34 @@ class Collection(BaseCollection):
|
||||
|
||||
def get_meta(self, key=None):
|
||||
if os.path.exists(self._props_path):
|
||||
with open(self._props_path, encoding=self.encoding) as prop:
|
||||
meta = json.load(prop)
|
||||
with open(self._props_path, encoding=self.encoding) as f:
|
||||
meta = json.load(f)
|
||||
return meta.get(key) if key else meta
|
||||
|
||||
def set_meta(self, props):
|
||||
if os.path.exists(self._props_path):
|
||||
with open(self._props_path, encoding=self.encoding) as prop:
|
||||
old_props = json.load(prop)
|
||||
with open(self._props_path, encoding=self.encoding) as f:
|
||||
old_props = json.load(f)
|
||||
old_props.update(props)
|
||||
props = old_props
|
||||
props = {key: value for key, value in props.items() if value}
|
||||
with self._atomic_write(self._props_path, "w+") as prop:
|
||||
json.dump(props, prop)
|
||||
with self._atomic_write(self._props_path, "w+") as f:
|
||||
json.dump(props, f)
|
||||
|
||||
@property
|
||||
def last_modified(self):
|
||||
last = max([os.path.getmtime(self._filesystem_path)] + [
|
||||
os.path.getmtime(os.path.join(self._filesystem_path, filename))
|
||||
for filename in os.listdir(self._filesystem_path)] or [0])
|
||||
relevant_files = [self._filesystem_path] + [
|
||||
path_to_filesystem(self._filesystem_path, href)
|
||||
for href in self.list()]
|
||||
if os.path.exists(self._props_path):
|
||||
relevant_files.append(self._props_path)
|
||||
last = max(map(os.path.getmtime, relevant_files))
|
||||
return time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last))
|
||||
|
||||
def serialize(self):
|
||||
if not os.path.exists(self._filesystem_path):
|
||||
return None
|
||||
items = []
|
||||
for href in os.listdir(self._filesystem_path):
|
||||
if not is_safe_filesystem_path_component(href):
|
||||
self.logger.debug("Skipping component: %s", href)
|
||||
continue
|
||||
path = os.path.join(self._filesystem_path, href)
|
||||
if os.path.isfile(path):
|
||||
self.logger.debug("Read object: %s", path)
|
||||
with open(path, encoding=self.encoding, newline="") as fd:
|
||||
items.append(vobject.readOne(fd.read()))
|
||||
for href in self.list():
|
||||
items.append(self.get(href).item)
|
||||
if self.get_meta("tag") == "VCALENDAR":
|
||||
collection = vobject.iCalendar()
|
||||
for item in items:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user