tobias/docker-offlineimap
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

SQLite: make postponing transaction committing possible.

Browse Source 
This should significantly improve performance when used to write large
amounts of messages.

This feature is enabled through the fsync configuration option.

Code refactorize around fsync.

This addresses #390 (although it doesn't necessarily fix all instances
of that problem yet).

Github-ref: https://github.com/OfflineIMAP/offlineimap/issues/390
Originally-written-by: Giel van Schijndel <me@mortis.eu>
Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
This commit is contained in:
Giel van Schijndel
2016-10-25 21:34:38 +02:00
committed by Nicolas Sebrecht
parent 41c9694488
commit b6ede627a9
5 changed files with 65 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
offlineimap/folder/Base.py
View File

@ -70,6 +70,7 @@ class BaseFolder(object):
self._sync_deletes = self.config.getdefaultboolean(
self.repoconfname, "sync_deletes", True)
self._dofsync = self.config.getdefaultboolean("general", "fsync", True)
# Determine if we're running static or dynamic folder filtering
# and check filtering status.
@ -103,6 +104,16 @@ class BaseFolder(object):
# fails if the str is utf-8
return self.name.decode('utf-8')
def __enter__(self):
"""Starts a transaction. This will postpone (guaranteed) saving to disk
of all messages saved inside this transaction until its committed."""
pass
def __exit__(self, exc_type, exc_val, exc_tb):
"""Commits a transaction, all messages saved inside this transaction
will only now be persisted to disk."""
pass
@property
def accountname(self):
"""Account name as string"""
@ -118,6 +129,9 @@ class BaseFolder(object):
else:
return self.repository.should_sync_folder(self.ffilter_name)
def dofsync(self):
return self._dofsync
def suggeststhreads(self):
"""Returns True if this folder suggests using threads for actions.
@ -891,38 +905,39 @@ class BaseFolder(object):
)
return
for num, uid in enumerate(copylist):
# Bail out on CTRL-C or SIGTERM.
if offlineimap.accounts.Account.abort_NOW_signal.is_set():
break
with self:
for num, uid in enumerate(copylist):
# Bail out on CTRL-C or SIGTERM.
if offlineimap.accounts.Account.abort_NOW_signal.is_set():
break
if uid == 0:
self.ui.warn("Assertion that UID != 0 failed; ignoring message.")
continue
if uid == 0:
self.ui.warn("Assertion that UID != 0 failed; ignoring message.")
continue
if uid > 0 and dstfolder.uidexists(uid):
# dstfolder has message with that UID already, only update status.
flags = self.getmessageflags(uid)
rtime = self.getmessagetime(uid)
statusfolder.savemessage(uid, None, flags, rtime)
continue
if uid > 0 and dstfolder.uidexists(uid):
# dstfolder has message with that UID already, only update status.
flags = self.getmessageflags(uid)
rtime = self.getmessagetime(uid)
statusfolder.savemessage(uid, None, flags, rtime)
continue
self.ui.copyingmessage(uid, num+1, num_to_copy, self, dstfolder)
# Exceptions are caught in copymessageto().
if self.suggeststhreads():
self.waitforthread()
thread = threadutil.InstanceLimitedThread(
self.getinstancelimitnamespace(),
target=self.copymessageto,
name="Copy message from %s:%s"% (self.repository, self),
args=(uid, dstfolder, statusfolder)
)
thread.start()
threads.append(thread)
else:
self.copymessageto(uid, dstfolder, statusfolder, register=0)
for thread in threads:
thread.join() # Block until all "copy" threads are done.
self.ui.copyingmessage(uid, num+1, num_to_copy, self, dstfolder)
# Exceptions are caught in copymessageto().
if self.suggeststhreads():
self.waitforthread()
thread = threadutil.InstanceLimitedThread(
self.getinstancelimitnamespace(),
target=self.copymessageto,
name="Copy message from %s:%s"% (self.repository, self),
args=(uid, dstfolder, statusfolder)
)
thread.start()
threads.append(thread)
else:
self.copymessageto(uid, dstfolder, statusfolder, register=0)
for thread in threads:
thread.join() # Block until all "copy" threads are done.
# Execute new mail hook if we have new mail.
if self.have_newmail: