diff --git a/radicale/storage.py b/radicale/storage.py index 1ea396f..80c28f9 100644 --- a/radicale/storage.py +++ b/radicale/storage.py @@ -100,7 +100,11 @@ def load(configuration, logger): def get_etag(text): - """Etag from collection or item.""" + """Etag from collection or item. + + Encoded as quoted-string (see RFC 2616). + + """ etag = md5() etag.update(text.encode("utf-8")) return '"%s"' % etag.hexdigest() @@ -121,7 +125,7 @@ def sanitize_path(path): path = posixpath.normpath(path) new_path = "/" for part in path.split("/"): - if not part or part in (".", ".."): + if not is_safe_path_component(part): continue new_path = posixpath.join(new_path, part) trailing_slash = "" if new_path.endswith("/") else trailing_slash @@ -138,7 +142,8 @@ def is_safe_path_component(path): def is_safe_filesystem_path_component(path): - """Check if path is a single component of a filesystem path. + """Check if path is a single component of a local and posix filesystem + path. Check that the path is safe to join too. @@ -146,7 +151,8 @@ def is_safe_filesystem_path_component(path): return ( path and not os.path.splitdrive(path)[0] and not os.path.split(path)[0] and path not in (os.curdir, os.pardir) and - not path.startswith(".") and not path.endswith("~")) + not path.startswith(".") and not path.endswith("~") and + is_safe_path_component(path)) def path_to_filesystem(root, *paths): @@ -187,12 +193,6 @@ class ComponentNotFoundError(ValueError): super().__init__(message) -class EtagMismatchError(ValueError): - def __init__(self, etag1, etag2): - message = "ETags don't match: %s != %s" % (etag1, etag2) - super().__init__(message) - - class Item: def __init__(self, collection, item, href, last_modified=None): self.collection = collection @@ -205,6 +205,7 @@ class Item: @property def etag(self): + """Encoded as quoted-string (see RFC 2616).""" return get_etag(self.serialize()) @@ -259,6 +260,7 @@ class BaseCollection: @property def etag(self): + """Encoded as quoted-string (see RFC 2616).""" return get_etag(self.serialize()) @classmethod @@ -389,11 +391,7 @@ class Collection(BaseCollection): delete=False, prefix=".Radicale.tmp-", newline=newline) try: yield tmp - if self.configuration.getboolean("storage", "filesystem_fsync"): - if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"): - fcntl.fcntl(tmp.fileno(), fcntl.F_FULLFSYNC) - else: - os.fsync(tmp.fileno()) + self._fsync(tmp.fileno()) tmp.close() os.replace(tmp.name, path) except: @@ -411,6 +409,14 @@ class Collection(BaseCollection): return file_name raise FileExistsError(errno.EEXIST, "No usable file name found") + @classmethod + def _fsync(cls, fd): + if cls.configuration.getboolean("storage", "filesystem_fsync"): + if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"): + fcntl.fcntl(fd, fcntl.F_FULLFSYNC) + else: + os.fsync(fd) + @classmethod def _sync_directory(cls, path): """Sync directory to disk. @@ -423,10 +429,7 @@ class Collection(BaseCollection): if os.name == "posix": fd = os.open(path, 0) try: - if hasattr(fcntl, "F_FULLFSYNC"): - fcntl.fcntl(fd, fcntl.F_FULLFSYNC) - else: - os.fsync(fd) + cls._fsync(fd) finally: os.close(fd) @@ -581,23 +584,21 @@ class Collection(BaseCollection): """ fs = [] for href, item in vobject_items.items(): + if not is_safe_filesystem_path_component(href): + raise UnsafePathError(href) path = path_to_filesystem(self._filesystem_path, href) fs.append(open(path, "w", encoding=self.encoding, newline="")) fs[-1].write(item.serialize()) - fsync_fn = lambda fd: None - if self.configuration.getboolean("storage", "filesystem_fsync"): - if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"): - fsync_fn = lambda fd: fcntl.fcntl(fd, fcntl.F_FULLFSYNC) - else: - fsync_fn = os.fsync # sync everything at once because it's slightly faster. for f in fs: - fsync_fn(f.fileno()) + self._fsync(f.fileno()) f.close() self._sync_directory(self._filesystem_path) @classmethod def move(cls, item, to_collection, to_href): + if not is_safe_filesystem_path_component(to_href): + raise UnsafePathError(to_href) os.replace( path_to_filesystem(item.collection._filesystem_path, item.href), path_to_filesystem(to_collection._filesystem_path, to_href)) @@ -606,12 +607,7 @@ class Collection(BaseCollection): cls._sync_directory(item.collection._filesystem_path) def list(self): - try: - hrefs = os.listdir(self._filesystem_path) - except IOError: - return - - for href in hrefs: + for href in os.listdir(self._filesystem_path): if not is_safe_filesystem_path_component(href): if not href.startswith(".Radicale"): self.logger.debug("Skipping component: %s", href) @@ -623,7 +619,6 @@ class Collection(BaseCollection): def get(self, href): if not href: return None - href = href.strip("{}").replace("/", "_") if not is_safe_filesystem_path_component(href): self.logger.debug( "Can't translate name safely to filesystem: %s", href) @@ -631,15 +626,17 @@ class Collection(BaseCollection): path = path_to_filesystem(self._filesystem_path, href) if not os.path.isfile(path): return None - with open(path, encoding=self.encoding, newline="") as fd: - text = fd.read() + with open(path, encoding=self.encoding, newline="") as f: + text = f.read() last_modified = time.strftime( "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(os.path.getmtime(path))) - return Item(self, vobject.readOne(text), href, last_modified) - - def has(self, href): - return self.get(href) is not None + try: + item = vobject.readOne(text) + except Exception: + self.logger.error("Failed to parse component: %s", href) + raise + return Item(self, item, href, last_modified) def upload(self, href, vobject_item): if not is_safe_filesystem_path_component(href): @@ -653,18 +650,17 @@ class Collection(BaseCollection): def delete(self, href=None): if href is None: # Delete the collection - if os.path.isdir(self._filesystem_path): - parent_dir = os.path.dirname(self._filesystem_path) - try: - os.rmdir(self._filesystem_path) - except OSError: - with TemporaryDirectory( - prefix=".Radicale.tmp-", dir=parent_dir) as tmp: - os.rename(self._filesystem_path, os.path.join( - tmp, os.path.basename(self._filesystem_path))) - self._sync_directory(parent_dir) - else: + parent_dir = os.path.dirname(self._filesystem_path) + try: + os.rmdir(self._filesystem_path) + except OSError: + with TemporaryDirectory( + prefix=".Radicale.tmp-", dir=parent_dir) as tmp: + os.rename(self._filesystem_path, os.path.join( + tmp, os.path.basename(self._filesystem_path))) self._sync_directory(parent_dir) + else: + self._sync_directory(parent_dir) else: # Delete an item if not is_safe_filesystem_path_component(href): @@ -677,40 +673,34 @@ class Collection(BaseCollection): def get_meta(self, key=None): if os.path.exists(self._props_path): - with open(self._props_path, encoding=self.encoding) as prop: - meta = json.load(prop) + with open(self._props_path, encoding=self.encoding) as f: + meta = json.load(f) return meta.get(key) if key else meta def set_meta(self, props): if os.path.exists(self._props_path): - with open(self._props_path, encoding=self.encoding) as prop: - old_props = json.load(prop) + with open(self._props_path, encoding=self.encoding) as f: + old_props = json.load(f) old_props.update(props) props = old_props props = {key: value for key, value in props.items() if value} - with self._atomic_write(self._props_path, "w+") as prop: - json.dump(props, prop) + with self._atomic_write(self._props_path, "w+") as f: + json.dump(props, f) @property def last_modified(self): - last = max([os.path.getmtime(self._filesystem_path)] + [ - os.path.getmtime(os.path.join(self._filesystem_path, filename)) - for filename in os.listdir(self._filesystem_path)] or [0]) + relevant_files = [self._filesystem_path] + [ + path_to_filesystem(self._filesystem_path, href) + for href in self.list()] + if os.path.exists(self._props_path): + relevant_files.append(self._props_path) + last = max(map(os.path.getmtime, relevant_files)) return time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last)) def serialize(self): - if not os.path.exists(self._filesystem_path): - return None items = [] - for href in os.listdir(self._filesystem_path): - if not is_safe_filesystem_path_component(href): - self.logger.debug("Skipping component: %s", href) - continue - path = os.path.join(self._filesystem_path, href) - if os.path.isfile(path): - self.logger.debug("Read object: %s", path) - with open(path, encoding=self.encoding, newline="") as fd: - items.append(vobject.readOne(fd.read())) + for href in self.list(): + items.append(self.get(href).item) if self.get_meta("tag") == "VCALENDAR": collection = vobject.iCalendar() for item in items: