tobias/radicale
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'storage' of https://github.com/Unrud/Radicale into Unrud-storage

Browse Source
This commit is contained in:
Guillaume Ayoub 2017-02-26 15:47:59 +01:00
commit 04764c2af4
1 changed files with 60 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
radicale/storage.py
View File

@ -100,7 +100,11 @@ def load(configuration, logger):
def get_etag(text): def get_etag(text):
"""Etag from collection or item.""" """Etag from collection or item.
Encoded as quoted-string (see RFC 2616).
"""
etag = md5() etag = md5()
etag.update(text.encode("utf-8")) etag.update(text.encode("utf-8"))
return '"%s"' % etag.hexdigest() return '"%s"' % etag.hexdigest()
@ -121,7 +125,7 @@ def sanitize_path(path):
path = posixpath.normpath(path) path = posixpath.normpath(path)
new_path = "/" new_path = "/"
for part in path.split("/"): for part in path.split("/"):
if not part or part in (".", ".."): if not is_safe_path_component(part):
continue continue
new_path = posixpath.join(new_path, part) new_path = posixpath.join(new_path, part)
trailing_slash = "" if new_path.endswith("/") else trailing_slash trailing_slash = "" if new_path.endswith("/") else trailing_slash
@ -138,7 +142,8 @@ def is_safe_path_component(path):
def is_safe_filesystem_path_component(path): def is_safe_filesystem_path_component(path):
"""Check if path is a single component of a filesystem path. """Check if path is a single component of a local and posix filesystem
path.
Check that the path is safe to join too. Check that the path is safe to join too.
@ -146,7 +151,8 @@ def is_safe_filesystem_path_component(path):
return ( return (
path and not os.path.splitdrive(path)[0] and path and not os.path.splitdrive(path)[0] and
not os.path.split(path)[0] and path not in (os.curdir, os.pardir) and not os.path.split(path)[0] and path not in (os.curdir, os.pardir) and
not path.startswith(".") and not path.endswith("~")) not path.startswith(".") and not path.endswith("~") and
is_safe_path_component(path))
def path_to_filesystem(root, *paths): def path_to_filesystem(root, *paths):
@ -187,12 +193,6 @@ class ComponentNotFoundError(ValueError):
super().__init__(message) super().__init__(message)
class EtagMismatchError(ValueError):
def __init__(self, etag1, etag2):
message = "ETags don't match: %s != %s" % (etag1, etag2)
super().__init__(message)
class Item: class Item:
def __init__(self, collection, item, href, last_modified=None): def __init__(self, collection, item, href, last_modified=None):
self.collection = collection self.collection = collection
@ -205,6 +205,7 @@ class Item:
@property @property
def etag(self): def etag(self):
"""Encoded as quoted-string (see RFC 2616)."""
return get_etag(self.serialize()) return get_etag(self.serialize())
@ -259,6 +260,7 @@ class BaseCollection:
@property @property
def etag(self): def etag(self):
"""Encoded as quoted-string (see RFC 2616)."""
return get_etag(self.serialize()) return get_etag(self.serialize())
@classmethod @classmethod
@ -389,11 +391,7 @@ class Collection(BaseCollection):
delete=False, prefix=".Radicale.tmp-", newline=newline) delete=False, prefix=".Radicale.tmp-", newline=newline)
try: try:
yield tmp yield tmp
if self.configuration.getboolean("storage", "filesystem_fsync"): self._fsync(tmp.fileno())
if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"):
fcntl.fcntl(tmp.fileno(), fcntl.F_FULLFSYNC)
else:
os.fsync(tmp.fileno())
tmp.close() tmp.close()
os.replace(tmp.name, path) os.replace(tmp.name, path)
except: except:
@ -411,6 +409,14 @@ class Collection(BaseCollection):
return file_name return file_name
raise FileExistsError(errno.EEXIST, "No usable file name found") raise FileExistsError(errno.EEXIST, "No usable file name found")
@classmethod
def _fsync(cls, fd):
if cls.configuration.getboolean("storage", "filesystem_fsync"):
if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"):
fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
else:
os.fsync(fd)
@classmethod @classmethod
def _sync_directory(cls, path): def _sync_directory(cls, path):
"""Sync directory to disk. """Sync directory to disk.
@ -423,10 +429,7 @@ class Collection(BaseCollection):
if os.name == "posix": if os.name == "posix":
fd = os.open(path, 0) fd = os.open(path, 0)
try: try:
if hasattr(fcntl, "F_FULLFSYNC"): cls._fsync(fd)
fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
else:
os.fsync(fd)
finally: finally:
os.close(fd) os.close(fd)
@ -581,23 +584,21 @@ class Collection(BaseCollection):
""" """
fs = [] fs = []
for href, item in vobject_items.items(): for href, item in vobject_items.items():
if not is_safe_filesystem_path_component(href):
raise UnsafePathError(href)
path = path_to_filesystem(self._filesystem_path, href) path = path_to_filesystem(self._filesystem_path, href)
fs.append(open(path, "w", encoding=self.encoding, newline="")) fs.append(open(path, "w", encoding=self.encoding, newline=""))
fs[-1].write(item.serialize()) fs[-1].write(item.serialize())
fsync_fn = lambda fd: None
if self.configuration.getboolean("storage", "filesystem_fsync"):
if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"):
fsync_fn = lambda fd: fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
else:
fsync_fn = os.fsync
# sync everything at once because it's slightly faster. # sync everything at once because it's slightly faster.
for f in fs: for f in fs:
fsync_fn(f.fileno()) self._fsync(f.fileno())
f.close() f.close()
self._sync_directory(self._filesystem_path) self._sync_directory(self._filesystem_path)
@classmethod @classmethod
def move(cls, item, to_collection, to_href): def move(cls, item, to_collection, to_href):
if not is_safe_filesystem_path_component(to_href):
raise UnsafePathError(to_href)
os.replace( os.replace(
path_to_filesystem(item.collection._filesystem_path, item.href), path_to_filesystem(item.collection._filesystem_path, item.href),
path_to_filesystem(to_collection._filesystem_path, to_href)) path_to_filesystem(to_collection._filesystem_path, to_href))
@ -606,12 +607,7 @@ class Collection(BaseCollection):
cls._sync_directory(item.collection._filesystem_path) cls._sync_directory(item.collection._filesystem_path)
def list(self): def list(self):
try: for href in os.listdir(self._filesystem_path):
hrefs = os.listdir(self._filesystem_path)
except IOError:
return
for href in hrefs:
if not is_safe_filesystem_path_component(href): if not is_safe_filesystem_path_component(href):
if not href.startswith(".Radicale"): if not href.startswith(".Radicale"):
self.logger.debug("Skipping component: %s", href) self.logger.debug("Skipping component: %s", href)
@ -623,7 +619,6 @@ class Collection(BaseCollection):
def get(self, href): def get(self, href):
if not href: if not href:
return None return None
href = href.strip("{}").replace("/", "_")
if not is_safe_filesystem_path_component(href): if not is_safe_filesystem_path_component(href):
self.logger.debug( self.logger.debug(
"Can't translate name safely to filesystem: %s", href) "Can't translate name safely to filesystem: %s", href)
@ -631,15 +626,17 @@ class Collection(BaseCollection):
path = path_to_filesystem(self._filesystem_path, href) path = path_to_filesystem(self._filesystem_path, href)
if not os.path.isfile(path): if not os.path.isfile(path):
return None return None
with open(path, encoding=self.encoding, newline="") as fd: with open(path, encoding=self.encoding, newline="") as f:
text = fd.read() text = f.read()
last_modified = time.strftime( last_modified = time.strftime(
"%a, %d %b %Y %H:%M:%S GMT", "%a, %d %b %Y %H:%M:%S GMT",
time.gmtime(os.path.getmtime(path))) time.gmtime(os.path.getmtime(path)))
return Item(self, vobject.readOne(text), href, last_modified) try:
item = vobject.readOne(text)
def has(self, href): except Exception:
return self.get(href) is not None self.logger.error("Failed to parse component: %s", href)
raise
return Item(self, item, href, last_modified)
def upload(self, href, vobject_item): def upload(self, href, vobject_item):
if not is_safe_filesystem_path_component(href): if not is_safe_filesystem_path_component(href):
@ -653,18 +650,17 @@ class Collection(BaseCollection):
def delete(self, href=None): def delete(self, href=None):
if href is None: if href is None:
# Delete the collection # Delete the collection
if os.path.isdir(self._filesystem_path): parent_dir = os.path.dirname(self._filesystem_path)
parent_dir = os.path.dirname(self._filesystem_path) try:
try: os.rmdir(self._filesystem_path)
os.rmdir(self._filesystem_path) except OSError:
except OSError: with TemporaryDirectory(
with TemporaryDirectory( prefix=".Radicale.tmp-", dir=parent_dir) as tmp:
prefix=".Radicale.tmp-", dir=parent_dir) as tmp: os.rename(self._filesystem_path, os.path.join(
os.rename(self._filesystem_path, os.path.join( tmp, os.path.basename(self._filesystem_path)))
tmp, os.path.basename(self._filesystem_path)))
self._sync_directory(parent_dir)
else:
self._sync_directory(parent_dir) self._sync_directory(parent_dir)
else:
self._sync_directory(parent_dir)
else: else:
# Delete an item # Delete an item
if not is_safe_filesystem_path_component(href): if not is_safe_filesystem_path_component(href):
@ -677,40 +673,34 @@ class Collection(BaseCollection):
def get_meta(self, key=None): def get_meta(self, key=None):
if os.path.exists(self._props_path): if os.path.exists(self._props_path):
with open(self._props_path, encoding=self.encoding) as prop: with open(self._props_path, encoding=self.encoding) as f:
meta = json.load(prop) meta = json.load(f)
return meta.get(key) if key else meta return meta.get(key) if key else meta
def set_meta(self, props): def set_meta(self, props):
if os.path.exists(self._props_path): if os.path.exists(self._props_path):
with open(self._props_path, encoding=self.encoding) as prop: with open(self._props_path, encoding=self.encoding) as f:
old_props = json.load(prop) old_props = json.load(f)
old_props.update(props) old_props.update(props)
props = old_props props = old_props
props = {key: value for key, value in props.items() if value} props = {key: value for key, value in props.items() if value}
with self._atomic_write(self._props_path, "w+") as prop: with self._atomic_write(self._props_path, "w+") as f:
json.dump(props, prop) json.dump(props, f)
@property @property
def last_modified(self): def last_modified(self):
last = max([os.path.getmtime(self._filesystem_path)] + [ relevant_files = [self._filesystem_path] + [
os.path.getmtime(os.path.join(self._filesystem_path, filename)) path_to_filesystem(self._filesystem_path, href)
for filename in os.listdir(self._filesystem_path)] or [0]) for href in self.list()]
if os.path.exists(self._props_path):
relevant_files.append(self._props_path)
last = max(map(os.path.getmtime, relevant_files))
return time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last)) return time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last))
def serialize(self): def serialize(self):
if not os.path.exists(self._filesystem_path):
return None
items = [] items = []
for href in os.listdir(self._filesystem_path): for href in self.list():
if not is_safe_filesystem_path_component(href): items.append(self.get(href).item)
self.logger.debug("Skipping component: %s", href)
continue
path = os.path.join(self._filesystem_path, href)
if os.path.isfile(path):
self.logger.debug("Read object: %s", path)
with open(path, encoding=self.encoding, newline="") as fd:
items.append(vobject.readOne(fd.read()))
if self.get_meta("tag") == "VCALENDAR": if self.get_meta("tag") == "VCALENDAR":
collection = vobject.iCalendar() collection = vobject.iCalendar()
for item in items: for item in items: