Merge branch 'storage' of https://github.com/Unrud/Radicale into Unrud-storage
This commit is contained in:
commit
04764c2af4
@ -100,7 +100,11 @@ def load(configuration, logger):
|
|||||||
|
|
||||||
|
|
||||||
def get_etag(text):
|
def get_etag(text):
|
||||||
"""Etag from collection or item."""
|
"""Etag from collection or item.
|
||||||
|
|
||||||
|
Encoded as quoted-string (see RFC 2616).
|
||||||
|
|
||||||
|
"""
|
||||||
etag = md5()
|
etag = md5()
|
||||||
etag.update(text.encode("utf-8"))
|
etag.update(text.encode("utf-8"))
|
||||||
return '"%s"' % etag.hexdigest()
|
return '"%s"' % etag.hexdigest()
|
||||||
@ -121,7 +125,7 @@ def sanitize_path(path):
|
|||||||
path = posixpath.normpath(path)
|
path = posixpath.normpath(path)
|
||||||
new_path = "/"
|
new_path = "/"
|
||||||
for part in path.split("/"):
|
for part in path.split("/"):
|
||||||
if not part or part in (".", ".."):
|
if not is_safe_path_component(part):
|
||||||
continue
|
continue
|
||||||
new_path = posixpath.join(new_path, part)
|
new_path = posixpath.join(new_path, part)
|
||||||
trailing_slash = "" if new_path.endswith("/") else trailing_slash
|
trailing_slash = "" if new_path.endswith("/") else trailing_slash
|
||||||
@ -138,7 +142,8 @@ def is_safe_path_component(path):
|
|||||||
|
|
||||||
|
|
||||||
def is_safe_filesystem_path_component(path):
|
def is_safe_filesystem_path_component(path):
|
||||||
"""Check if path is a single component of a filesystem path.
|
"""Check if path is a single component of a local and posix filesystem
|
||||||
|
path.
|
||||||
|
|
||||||
Check that the path is safe to join too.
|
Check that the path is safe to join too.
|
||||||
|
|
||||||
@ -146,7 +151,8 @@ def is_safe_filesystem_path_component(path):
|
|||||||
return (
|
return (
|
||||||
path and not os.path.splitdrive(path)[0] and
|
path and not os.path.splitdrive(path)[0] and
|
||||||
not os.path.split(path)[0] and path not in (os.curdir, os.pardir) and
|
not os.path.split(path)[0] and path not in (os.curdir, os.pardir) and
|
||||||
not path.startswith(".") and not path.endswith("~"))
|
not path.startswith(".") and not path.endswith("~") and
|
||||||
|
is_safe_path_component(path))
|
||||||
|
|
||||||
|
|
||||||
def path_to_filesystem(root, *paths):
|
def path_to_filesystem(root, *paths):
|
||||||
@ -187,12 +193,6 @@ class ComponentNotFoundError(ValueError):
|
|||||||
super().__init__(message)
|
super().__init__(message)
|
||||||
|
|
||||||
|
|
||||||
class EtagMismatchError(ValueError):
|
|
||||||
def __init__(self, etag1, etag2):
|
|
||||||
message = "ETags don't match: %s != %s" % (etag1, etag2)
|
|
||||||
super().__init__(message)
|
|
||||||
|
|
||||||
|
|
||||||
class Item:
|
class Item:
|
||||||
def __init__(self, collection, item, href, last_modified=None):
|
def __init__(self, collection, item, href, last_modified=None):
|
||||||
self.collection = collection
|
self.collection = collection
|
||||||
@ -205,6 +205,7 @@ class Item:
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def etag(self):
|
def etag(self):
|
||||||
|
"""Encoded as quoted-string (see RFC 2616)."""
|
||||||
return get_etag(self.serialize())
|
return get_etag(self.serialize())
|
||||||
|
|
||||||
|
|
||||||
@ -259,6 +260,7 @@ class BaseCollection:
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def etag(self):
|
def etag(self):
|
||||||
|
"""Encoded as quoted-string (see RFC 2616)."""
|
||||||
return get_etag(self.serialize())
|
return get_etag(self.serialize())
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@ -389,11 +391,7 @@ class Collection(BaseCollection):
|
|||||||
delete=False, prefix=".Radicale.tmp-", newline=newline)
|
delete=False, prefix=".Radicale.tmp-", newline=newline)
|
||||||
try:
|
try:
|
||||||
yield tmp
|
yield tmp
|
||||||
if self.configuration.getboolean("storage", "filesystem_fsync"):
|
self._fsync(tmp.fileno())
|
||||||
if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"):
|
|
||||||
fcntl.fcntl(tmp.fileno(), fcntl.F_FULLFSYNC)
|
|
||||||
else:
|
|
||||||
os.fsync(tmp.fileno())
|
|
||||||
tmp.close()
|
tmp.close()
|
||||||
os.replace(tmp.name, path)
|
os.replace(tmp.name, path)
|
||||||
except:
|
except:
|
||||||
@ -411,6 +409,14 @@ class Collection(BaseCollection):
|
|||||||
return file_name
|
return file_name
|
||||||
raise FileExistsError(errno.EEXIST, "No usable file name found")
|
raise FileExistsError(errno.EEXIST, "No usable file name found")
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _fsync(cls, fd):
|
||||||
|
if cls.configuration.getboolean("storage", "filesystem_fsync"):
|
||||||
|
if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"):
|
||||||
|
fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
|
||||||
|
else:
|
||||||
|
os.fsync(fd)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _sync_directory(cls, path):
|
def _sync_directory(cls, path):
|
||||||
"""Sync directory to disk.
|
"""Sync directory to disk.
|
||||||
@ -423,10 +429,7 @@ class Collection(BaseCollection):
|
|||||||
if os.name == "posix":
|
if os.name == "posix":
|
||||||
fd = os.open(path, 0)
|
fd = os.open(path, 0)
|
||||||
try:
|
try:
|
||||||
if hasattr(fcntl, "F_FULLFSYNC"):
|
cls._fsync(fd)
|
||||||
fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
|
|
||||||
else:
|
|
||||||
os.fsync(fd)
|
|
||||||
finally:
|
finally:
|
||||||
os.close(fd)
|
os.close(fd)
|
||||||
|
|
||||||
@ -581,23 +584,21 @@ class Collection(BaseCollection):
|
|||||||
"""
|
"""
|
||||||
fs = []
|
fs = []
|
||||||
for href, item in vobject_items.items():
|
for href, item in vobject_items.items():
|
||||||
|
if not is_safe_filesystem_path_component(href):
|
||||||
|
raise UnsafePathError(href)
|
||||||
path = path_to_filesystem(self._filesystem_path, href)
|
path = path_to_filesystem(self._filesystem_path, href)
|
||||||
fs.append(open(path, "w", encoding=self.encoding, newline=""))
|
fs.append(open(path, "w", encoding=self.encoding, newline=""))
|
||||||
fs[-1].write(item.serialize())
|
fs[-1].write(item.serialize())
|
||||||
fsync_fn = lambda fd: None
|
|
||||||
if self.configuration.getboolean("storage", "filesystem_fsync"):
|
|
||||||
if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"):
|
|
||||||
fsync_fn = lambda fd: fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
|
|
||||||
else:
|
|
||||||
fsync_fn = os.fsync
|
|
||||||
# sync everything at once because it's slightly faster.
|
# sync everything at once because it's slightly faster.
|
||||||
for f in fs:
|
for f in fs:
|
||||||
fsync_fn(f.fileno())
|
self._fsync(f.fileno())
|
||||||
f.close()
|
f.close()
|
||||||
self._sync_directory(self._filesystem_path)
|
self._sync_directory(self._filesystem_path)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def move(cls, item, to_collection, to_href):
|
def move(cls, item, to_collection, to_href):
|
||||||
|
if not is_safe_filesystem_path_component(to_href):
|
||||||
|
raise UnsafePathError(to_href)
|
||||||
os.replace(
|
os.replace(
|
||||||
path_to_filesystem(item.collection._filesystem_path, item.href),
|
path_to_filesystem(item.collection._filesystem_path, item.href),
|
||||||
path_to_filesystem(to_collection._filesystem_path, to_href))
|
path_to_filesystem(to_collection._filesystem_path, to_href))
|
||||||
@ -606,12 +607,7 @@ class Collection(BaseCollection):
|
|||||||
cls._sync_directory(item.collection._filesystem_path)
|
cls._sync_directory(item.collection._filesystem_path)
|
||||||
|
|
||||||
def list(self):
|
def list(self):
|
||||||
try:
|
for href in os.listdir(self._filesystem_path):
|
||||||
hrefs = os.listdir(self._filesystem_path)
|
|
||||||
except IOError:
|
|
||||||
return
|
|
||||||
|
|
||||||
for href in hrefs:
|
|
||||||
if not is_safe_filesystem_path_component(href):
|
if not is_safe_filesystem_path_component(href):
|
||||||
if not href.startswith(".Radicale"):
|
if not href.startswith(".Radicale"):
|
||||||
self.logger.debug("Skipping component: %s", href)
|
self.logger.debug("Skipping component: %s", href)
|
||||||
@ -623,7 +619,6 @@ class Collection(BaseCollection):
|
|||||||
def get(self, href):
|
def get(self, href):
|
||||||
if not href:
|
if not href:
|
||||||
return None
|
return None
|
||||||
href = href.strip("{}").replace("/", "_")
|
|
||||||
if not is_safe_filesystem_path_component(href):
|
if not is_safe_filesystem_path_component(href):
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
"Can't translate name safely to filesystem: %s", href)
|
"Can't translate name safely to filesystem: %s", href)
|
||||||
@ -631,15 +626,17 @@ class Collection(BaseCollection):
|
|||||||
path = path_to_filesystem(self._filesystem_path, href)
|
path = path_to_filesystem(self._filesystem_path, href)
|
||||||
if not os.path.isfile(path):
|
if not os.path.isfile(path):
|
||||||
return None
|
return None
|
||||||
with open(path, encoding=self.encoding, newline="") as fd:
|
with open(path, encoding=self.encoding, newline="") as f:
|
||||||
text = fd.read()
|
text = f.read()
|
||||||
last_modified = time.strftime(
|
last_modified = time.strftime(
|
||||||
"%a, %d %b %Y %H:%M:%S GMT",
|
"%a, %d %b %Y %H:%M:%S GMT",
|
||||||
time.gmtime(os.path.getmtime(path)))
|
time.gmtime(os.path.getmtime(path)))
|
||||||
return Item(self, vobject.readOne(text), href, last_modified)
|
try:
|
||||||
|
item = vobject.readOne(text)
|
||||||
def has(self, href):
|
except Exception:
|
||||||
return self.get(href) is not None
|
self.logger.error("Failed to parse component: %s", href)
|
||||||
|
raise
|
||||||
|
return Item(self, item, href, last_modified)
|
||||||
|
|
||||||
def upload(self, href, vobject_item):
|
def upload(self, href, vobject_item):
|
||||||
if not is_safe_filesystem_path_component(href):
|
if not is_safe_filesystem_path_component(href):
|
||||||
@ -653,7 +650,6 @@ class Collection(BaseCollection):
|
|||||||
def delete(self, href=None):
|
def delete(self, href=None):
|
||||||
if href is None:
|
if href is None:
|
||||||
# Delete the collection
|
# Delete the collection
|
||||||
if os.path.isdir(self._filesystem_path):
|
|
||||||
parent_dir = os.path.dirname(self._filesystem_path)
|
parent_dir = os.path.dirname(self._filesystem_path)
|
||||||
try:
|
try:
|
||||||
os.rmdir(self._filesystem_path)
|
os.rmdir(self._filesystem_path)
|
||||||
@ -677,40 +673,34 @@ class Collection(BaseCollection):
|
|||||||
|
|
||||||
def get_meta(self, key=None):
|
def get_meta(self, key=None):
|
||||||
if os.path.exists(self._props_path):
|
if os.path.exists(self._props_path):
|
||||||
with open(self._props_path, encoding=self.encoding) as prop:
|
with open(self._props_path, encoding=self.encoding) as f:
|
||||||
meta = json.load(prop)
|
meta = json.load(f)
|
||||||
return meta.get(key) if key else meta
|
return meta.get(key) if key else meta
|
||||||
|
|
||||||
def set_meta(self, props):
|
def set_meta(self, props):
|
||||||
if os.path.exists(self._props_path):
|
if os.path.exists(self._props_path):
|
||||||
with open(self._props_path, encoding=self.encoding) as prop:
|
with open(self._props_path, encoding=self.encoding) as f:
|
||||||
old_props = json.load(prop)
|
old_props = json.load(f)
|
||||||
old_props.update(props)
|
old_props.update(props)
|
||||||
props = old_props
|
props = old_props
|
||||||
props = {key: value for key, value in props.items() if value}
|
props = {key: value for key, value in props.items() if value}
|
||||||
with self._atomic_write(self._props_path, "w+") as prop:
|
with self._atomic_write(self._props_path, "w+") as f:
|
||||||
json.dump(props, prop)
|
json.dump(props, f)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def last_modified(self):
|
def last_modified(self):
|
||||||
last = max([os.path.getmtime(self._filesystem_path)] + [
|
relevant_files = [self._filesystem_path] + [
|
||||||
os.path.getmtime(os.path.join(self._filesystem_path, filename))
|
path_to_filesystem(self._filesystem_path, href)
|
||||||
for filename in os.listdir(self._filesystem_path)] or [0])
|
for href in self.list()]
|
||||||
|
if os.path.exists(self._props_path):
|
||||||
|
relevant_files.append(self._props_path)
|
||||||
|
last = max(map(os.path.getmtime, relevant_files))
|
||||||
return time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last))
|
return time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last))
|
||||||
|
|
||||||
def serialize(self):
|
def serialize(self):
|
||||||
if not os.path.exists(self._filesystem_path):
|
|
||||||
return None
|
|
||||||
items = []
|
items = []
|
||||||
for href in os.listdir(self._filesystem_path):
|
for href in self.list():
|
||||||
if not is_safe_filesystem_path_component(href):
|
items.append(self.get(href).item)
|
||||||
self.logger.debug("Skipping component: %s", href)
|
|
||||||
continue
|
|
||||||
path = os.path.join(self._filesystem_path, href)
|
|
||||||
if os.path.isfile(path):
|
|
||||||
self.logger.debug("Read object: %s", path)
|
|
||||||
with open(path, encoding=self.encoding, newline="") as fd:
|
|
||||||
items.append(vobject.readOne(fd.read()))
|
|
||||||
if self.get_meta("tag") == "VCALENDAR":
|
if self.get_meta("tag") == "VCALENDAR":
|
||||||
collection = vobject.iCalendar()
|
collection = vobject.iCalendar()
|
||||||
for item in items:
|
for item in items:
|
||||||
|
Loading…
Reference in New Issue
Block a user