tobias/radicale
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Direct creation of files for batch uploads

Browse Source
This commit is contained in:
Unrud 2020-05-15 21:54:10 +02:00
parent 7b79c00ae2
commit 63e00ca677
2 changed files with 36 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
radicale/storage/multifilesystem/__init__.py
View File

@ -68,8 +68,7 @@ class Collection(
return self._path
@contextlib.contextmanager
def _atomic_write(self, path, mode="w", newline=None, sync_directory=True,
replace_fn=os.replace):
def _atomic_write(self, path, mode="w", newline=None):
directory = os.path.dirname(path)
tmp = NamedTemporaryFile(
mode=mode, dir=directory, delete=False, prefix=".Radicale.tmp-",
@ -77,18 +76,13 @@ class Collection(
try:
yield tmp
tmp.flush()
try:
self._storage._fsync(tmp.fileno())
except OSError as e:
raise RuntimeError("Fsync'ing file %r failed: %s" %
(path, e)) from e
self._storage._fsync(tmp)
tmp.close()
replace_fn(tmp.name, path)
os.replace(tmp.name, path)
except BaseException:
tmp.close()
os.remove(tmp.name)
raise
if sync_directory:
self._storage._sync_directory(directory)
@property
@ -124,9 +118,13 @@ class Storage(
"storage", "filesystem_folder")
return os.path.join(filesystem_folder, "collection-root")
def _fsync(self, fd):
def _fsync(self, f):
if self.configuration.get("storage", "_filesystem_fsync"):
pathutils.fsync(fd)
try:
pathutils.fsync(f.fileno())
except OSError as e:
raise RuntimeError("Fsync'ing file %r failed: %s" %
(f.name, e)) from e
def _sync_directory(self, path):
"""Sync directory to disk.
@ -140,7 +138,7 @@ class Storage(
try:
fd = os.open(path, 0)
try:
self._fsync(fd)
pathutils.fsync(fd)
finally:
os.close(fd)
except OSError as e:

26
radicale/storage/multifilesystem/upload.py
View File

@ -71,13 +71,9 @@ class CollectionUploadMixin:
lambda: radicale_item.get_etag(uid).strip('"') + suffix,
lambda: radicale_item.find_available_uid(hrefs.__contains__,
suffix)))
href = None
def replace_fn(source, target):
nonlocal href
href = f = None
while href_candidate_funtions:
href_fn = href_candidate_funtions.pop(0)
href = href_fn()
href = href_candidate_funtions.pop(0)()
if href in hrefs:
continue
if not pathutils.is_safe_filesystem_path_component(href):
@ -85,22 +81,24 @@ class CollectionUploadMixin:
raise pathutils.UnsafePathError(href)
continue
try:
return os.replace(source, pathutils.path_to_filesystem(
self._filesystem_path, href))
f = open(pathutils.path_to_filesystem(
self._filesystem_path, href),
"w", newline="", encoding=self._encoding)
break
except OSError as e:
if href_candidate_funtions and (
os.name == "posix" and e.errno == 22 or
os.name == "nt" and e.errno == 123):
continue
raise
with self._atomic_write(os.path.join(self._filesystem_path, "ign"),
newline="", sync_directory=False,
replace_fn=replace_fn) as f:
with f:
f.write(item.serialize())
f.flush()
self._storage._fsync(f)
hrefs.add(href)
with self._atomic_write(os.path.join(cache_folder, href), "wb",
sync_directory=False) as f:
with open(os.path.join(cache_folder, href), "wb") as f:
pickle.dump(cache_content, f)
f.flush()
self._storage._fsync(f)
self._storage._sync_directory(cache_folder)
self._storage._sync_directory(self._filesystem_path)