tobias/radicale
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'Unrud-storage'

Browse Source
This commit is contained in:
Guillaume Ayoub 2017-02-26 15:50:34 +01:00
commit b2e149ad0d
2 changed files with 67 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.gitignore vendored
View File

@ -1,12 +1,16 @@
*~
*.pyc
MANIFEST
build
dist
Radicale.egg-info
.cache
.coverage
.eggs
.project
.pydevproject
.settings
.tox
MANIFEST
.cache
.eggs

130
radicale/storage.py
View File

@ -100,7 +100,11 @@ def load(configuration, logger):
def get_etag(text):
"""Etag from collection or item."""
"""Etag from collection or item.
Encoded as quoted-string (see RFC 2616).
"""
etag = md5()
etag.update(text.encode("utf-8"))
return '"%s"' % etag.hexdigest()
@ -121,7 +125,7 @@ def sanitize_path(path):
path = posixpath.normpath(path)
new_path = "/"
for part in path.split("/"):
if not part or part in (".", ".."):
if not is_safe_path_component(part):
continue
new_path = posixpath.join(new_path, part)
trailing_slash = "" if new_path.endswith("/") else trailing_slash
@ -138,7 +142,8 @@ def is_safe_path_component(path):
def is_safe_filesystem_path_component(path):
"""Check if path is a single component of a filesystem path.
"""Check if path is a single component of a local and posix filesystem
path.
Check that the path is safe to join too.
@ -146,7 +151,8 @@ def is_safe_filesystem_path_component(path):
return (
path and not os.path.splitdrive(path)[0] and
not os.path.split(path)[0] and path not in (os.curdir, os.pardir) and
not path.startswith(".") and not path.endswith("~"))
not path.startswith(".") and not path.endswith("~") and
is_safe_path_component(path))
def path_to_filesystem(root, *paths):
@ -187,12 +193,6 @@ class ComponentNotFoundError(ValueError):
super().__init__(message)
class EtagMismatchError(ValueError):
def __init__(self, etag1, etag2):
message = "ETags don't match: %s != %s" % (etag1, etag2)
super().__init__(message)
class Item:
def __init__(self, collection, item, href, last_modified=None):
self.collection = collection
@ -205,6 +205,7 @@ class Item:
@property
def etag(self):
"""Encoded as quoted-string (see RFC 2616)."""
return get_etag(self.serialize())
@ -259,6 +260,7 @@ class BaseCollection:
@property
def etag(self):
"""Encoded as quoted-string (see RFC 2616)."""
return get_etag(self.serialize())
@classmethod
@ -389,11 +391,7 @@ class Collection(BaseCollection):
delete=False, prefix=".Radicale.tmp-", newline=newline)
try:
yield tmp
if self.configuration.getboolean("storage", "filesystem_fsync"):
if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"):
fcntl.fcntl(tmp.fileno(), fcntl.F_FULLFSYNC)
else:
os.fsync(tmp.fileno())
self._fsync(tmp.fileno())
tmp.close()
os.replace(tmp.name, path)
except:
@ -411,6 +409,14 @@ class Collection(BaseCollection):
return file_name
raise FileExistsError(errno.EEXIST, "No usable file name found")
@classmethod
def _fsync(cls, fd):
if cls.configuration.getboolean("storage", "filesystem_fsync"):
if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"):
fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
else:
os.fsync(fd)
@classmethod
def _sync_directory(cls, path):
"""Sync directory to disk.
@ -423,10 +429,7 @@ class Collection(BaseCollection):
if os.name == "posix":
fd = os.open(path, 0)
try:
if hasattr(fcntl, "F_FULLFSYNC"):
fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
else:
os.fsync(fd)
cls._fsync(fd)
finally:
os.close(fd)
@ -581,23 +584,21 @@ class Collection(BaseCollection):
"""
fs = []
for href, item in vobject_items.items():
if not is_safe_filesystem_path_component(href):
raise UnsafePathError(href)
path = path_to_filesystem(self._filesystem_path, href)
fs.append(open(path, "w", encoding=self.encoding, newline=""))
fs[-1].write(item.serialize())
fsync_fn = lambda fd: None
if self.configuration.getboolean("storage", "filesystem_fsync"):
if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"):
fsync_fn = lambda fd: fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
else:
fsync_fn = os.fsync
# sync everything at once because it's slightly faster.
for f in fs:
fsync_fn(f.fileno())
self._fsync(f.fileno())
f.close()
self._sync_directory(self._filesystem_path)
@classmethod
def move(cls, item, to_collection, to_href):
if not is_safe_filesystem_path_component(to_href):
raise UnsafePathError(to_href)
os.replace(
path_to_filesystem(item.collection._filesystem_path, item.href),
path_to_filesystem(to_collection._filesystem_path, to_href))
@ -606,12 +607,7 @@ class Collection(BaseCollection):
cls._sync_directory(item.collection._filesystem_path)
def list(self):
try:
hrefs = os.listdir(self._filesystem_path)
except IOError:
return
for href in hrefs:
for href in os.listdir(self._filesystem_path):
if not is_safe_filesystem_path_component(href):
if not href.startswith(".Radicale"):
self.logger.debug("Skipping component: %s", href)
@ -623,7 +619,6 @@ class Collection(BaseCollection):
def get(self, href):
if not href:
return None
href = href.strip("{}").replace("/", "_")
if not is_safe_filesystem_path_component(href):
self.logger.debug(
"Can't translate name safely to filesystem: %s", href)
@ -631,15 +626,17 @@ class Collection(BaseCollection):
path = path_to_filesystem(self._filesystem_path, href)
if not os.path.isfile(path):
return None
with open(path, encoding=self.encoding, newline="") as fd:
text = fd.read()
with open(path, encoding=self.encoding, newline="") as f:
text = f.read()
last_modified = time.strftime(
"%a, %d %b %Y %H:%M:%S GMT",
time.gmtime(os.path.getmtime(path)))
return Item(self, vobject.readOne(text), href, last_modified)
def has(self, href):
return self.get(href) is not None
try:
item = vobject.readOne(text)
except Exception:
self.logger.error("Failed to parse component: %s", href)
raise
return Item(self, item, href, last_modified)
def upload(self, href, vobject_item):
if not is_safe_filesystem_path_component(href):
@ -653,18 +650,17 @@ class Collection(BaseCollection):
def delete(self, href=None):
if href is None:
# Delete the collection
if os.path.isdir(self._filesystem_path):
parent_dir = os.path.dirname(self._filesystem_path)
try:
os.rmdir(self._filesystem_path)
except OSError:
with TemporaryDirectory(
prefix=".Radicale.tmp-", dir=parent_dir) as tmp:
os.rename(self._filesystem_path, os.path.join(
tmp, os.path.basename(self._filesystem_path)))
self._sync_directory(parent_dir)
else:
parent_dir = os.path.dirname(self._filesystem_path)
try:
os.rmdir(self._filesystem_path)
except OSError:
with TemporaryDirectory(
prefix=".Radicale.tmp-", dir=parent_dir) as tmp:
os.rename(self._filesystem_path, os.path.join(
tmp, os.path.basename(self._filesystem_path)))
self._sync_directory(parent_dir)
else:
self._sync_directory(parent_dir)
else:
# Delete an item
if not is_safe_filesystem_path_component(href):
@ -677,40 +673,34 @@ class Collection(BaseCollection):
def get_meta(self, key=None):
if os.path.exists(self._props_path):
with open(self._props_path, encoding=self.encoding) as prop:
meta = json.load(prop)
with open(self._props_path, encoding=self.encoding) as f:
meta = json.load(f)
return meta.get(key) if key else meta
def set_meta(self, props):
if os.path.exists(self._props_path):
with open(self._props_path, encoding=self.encoding) as prop:
old_props = json.load(prop)
with open(self._props_path, encoding=self.encoding) as f:
old_props = json.load(f)
old_props.update(props)
props = old_props
props = {key: value for key, value in props.items() if value}
with self._atomic_write(self._props_path, "w+") as prop:
json.dump(props, prop)
with self._atomic_write(self._props_path, "w+") as f:
json.dump(props, f)
@property
def last_modified(self):
last = max([os.path.getmtime(self._filesystem_path)] + [
os.path.getmtime(os.path.join(self._filesystem_path, filename))
for filename in os.listdir(self._filesystem_path)] or [0])
relevant_files = [self._filesystem_path] + [
path_to_filesystem(self._filesystem_path, href)
for href in self.list()]
if os.path.exists(self._props_path):
relevant_files.append(self._props_path)
last = max(map(os.path.getmtime, relevant_files))
return time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last))
def serialize(self):
if not os.path.exists(self._filesystem_path):
return None
items = []
for href in os.listdir(self._filesystem_path):
if not is_safe_filesystem_path_component(href):
self.logger.debug("Skipping component: %s", href)
continue
path = os.path.join(self._filesystem_path, href)
if os.path.isfile(path):
self.logger.debug("Read object: %s", path)
with open(path, encoding=self.encoding, newline="") as fd:
items.append(vobject.readOne(fd.read()))
for href in self.list():
items.append(self.get(href).item)
if self.get_meta("tag") == "VCALENDAR":
collection = vobject.iCalendar()
for item in items: