tobias/radicale
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Don't keep files open when uploading whole collection

Browse Source
This commit is contained in:
Unrud 2017-08-09 17:37:47 +02:00
parent 59eded976b
commit 1821b872d2
1 changed files with 23 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
radicale/storage.py
View File

@ -729,7 +729,7 @@ class Collection(BaseCollection):
return os.path.join(filesystem_folder, "collection-root") return os.path.join(filesystem_folder, "collection-root")
@contextmanager @contextmanager
def _atomic_write(self, path, mode="w", newline=None): def _atomic_write(self, path, mode="w", newline=None, sync_directory=True):
directory = os.path.dirname(path) directory = os.path.dirname(path)
tmp = NamedTemporaryFile( tmp = NamedTemporaryFile(
mode=mode, dir=directory, delete=False, prefix=".Radicale.tmp-", mode=mode, dir=directory, delete=False, prefix=".Radicale.tmp-",
@ -747,7 +747,8 @@ class Collection(BaseCollection):
tmp.close() tmp.close()
os.remove(tmp.name) os.remove(tmp.name)
raise raise
self._sync_directory(directory) if sync_directory:
self._sync_directory(directory)
@staticmethod @staticmethod
def _find_available_file_name(exists_fn, suffix=""): def _find_available_file_name(exists_fn, suffix=""):
@ -931,36 +932,26 @@ class Collection(BaseCollection):
uploads them nonatomic and without existence checks. uploads them nonatomic and without existence checks.
""" """
with contextlib.ExitStack() as stack: cache_folder = os.path.join(self._filesystem_path,
cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache", "item")
".Radicale.cache", "item") self._makedirs_synced(cache_folder)
self._makedirs_synced(cache_folder) for href, vobject_item in vobject_items.items():
fs = [] if not is_safe_filesystem_path_component(href):
for href, vobject_item in vobject_items.items(): raise UnsafePathError(href)
if not is_safe_filesystem_path_component(href): try:
raise UnsafePathError(href) cache_content = self._item_cache_content(href, vobject_item)
try: _, _, _, text, _, _, _ = cache_content
cache_content = self._item_cache_content(href, except Exception as e:
vobject_item) raise ValueError(
_, _, _, text, _, _, _ = cache_content "Failed to store item %r in temporary collection %r: %s" %
except Exception as e: (href, self.path, e)) from e
raise ValueError( with self._atomic_write(os.path.join(cache_folder, href), "wb",
"Failed to store item %r in temporary collection %r: " sync_directory=False) as f:
"%s" % (href, self.path, e)) from e pickle.dump(cache_content, f)
fs.append(stack.enter_context( path = path_to_filesystem(self._filesystem_path, href)
open(os.path.join(cache_folder, href), "wb"))) with self._atomic_write(
pickle.dump(cache_content, fs[-1]) path, newline="", sync_directory=False) as f:
path = path_to_filesystem(self._filesystem_path, href) f.write(text)
fs.append(stack.enter_context(
open(path, "w", encoding=self._encoding, newline="")))
fs[-1].write(text)
# sync everything at once because it's slightly faster.
for f in fs:
try:
self._fsync(f.fileno())
except OSError as e:
raise RuntimeError("Fsync'ing file %r failed: %s" %
(f.name, e)) from e
self._sync_directory(self._filesystem_path) self._sync_directory(self._filesystem_path)
@classmethod @classmethod