tobias/radicale
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Don't keep files open when uploading whole collection

Browse Source
This commit is contained in:
Unrud 2017-08-09 17:37:47 +02:00
parent 59eded976b
commit 1821b872d2
1 changed files with 23 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
radicale/storage.py
View File

@ -729,7 +729,7 @@ class Collection(BaseCollection):
return os.path.join(filesystem_folder, "collection-root") return os.path.join(filesystem_folder, "collection-root")
@contextmanager @contextmanager
def _atomic_write(self, path, mode="w", newline=None): def _atomic_write(self, path, mode="w", newline=None, sync_directory=True):
directory = os.path.dirname(path) directory = os.path.dirname(path)
tmp = NamedTemporaryFile( tmp = NamedTemporaryFile(
mode=mode, dir=directory, delete=False, prefix=".Radicale.tmp-", mode=mode, dir=directory, delete=False, prefix=".Radicale.tmp-",
@ -747,6 +747,7 @@ class Collection(BaseCollection):
tmp.close() tmp.close()
os.remove(tmp.name) os.remove(tmp.name)
raise raise
if sync_directory:
self._sync_directory(directory) self._sync_directory(directory)
@staticmethod @staticmethod
@ -931,36 +932,26 @@ class Collection(BaseCollection):
uploads them nonatomic and without existence checks. uploads them nonatomic and without existence checks.
""" """
with contextlib.ExitStack() as stack:
cache_folder = os.path.join(self._filesystem_path, cache_folder = os.path.join(self._filesystem_path,
".Radicale.cache", "item") ".Radicale.cache", "item")
self._makedirs_synced(cache_folder) self._makedirs_synced(cache_folder)
fs = []
for href, vobject_item in vobject_items.items(): for href, vobject_item in vobject_items.items():
if not is_safe_filesystem_path_component(href): if not is_safe_filesystem_path_component(href):
raise UnsafePathError(href) raise UnsafePathError(href)
try: try:
cache_content = self._item_cache_content(href, cache_content = self._item_cache_content(href, vobject_item)
vobject_item)
_, _, _, text, _, _, _ = cache_content _, _, _, text, _, _, _ = cache_content
except Exception as e: except Exception as e:
raise ValueError( raise ValueError(
"Failed to store item %r in temporary collection %r: " "Failed to store item %r in temporary collection %r: %s" %
"%s" % (href, self.path, e)) from e (href, self.path, e)) from e
fs.append(stack.enter_context( with self._atomic_write(os.path.join(cache_folder, href), "wb",
open(os.path.join(cache_folder, href), "wb"))) sync_directory=False) as f:
pickle.dump(cache_content, fs[-1]) pickle.dump(cache_content, f)
path = path_to_filesystem(self._filesystem_path, href) path = path_to_filesystem(self._filesystem_path, href)
fs.append(stack.enter_context( with self._atomic_write(
open(path, "w", encoding=self._encoding, newline=""))) path, newline="", sync_directory=False) as f:
fs[-1].write(text) f.write(text)
# sync everything at once because it's slightly faster.
for f in fs:
try:
self._fsync(f.fileno())
except OSError as e:
raise RuntimeError("Fsync'ing file %r failed: %s" %
(f.name, e)) from e
self._sync_directory(self._filesystem_path) self._sync_directory(self._filesystem_path)
@classmethod @classmethod