Improve delete performance with SQLITE backend

When deleting many (eg 2000) mails using the SQLITE backend, this takes
a long time durig which OfflineImap can not be aborted via
CTRL-C. Thinking it had frozen permanently, I killed it hard, leaving a
corrupted db journal (which leads to awkwards complaints by OLI on
subsequent starts!). That shows that delete performance is critical and
needs improvement.

We were iterating through the list of messages to delete and deleted
them one-by-one execute()'ing a new SQL Query for each message. This
patch improves the situation by allowing us to use executemany(), which
is -despite still being one SQL query per message- much faster. This is
because rather than performing a commit() after each mail, we now do
only one commit() after all mails have been deleted.

Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de>
This commit is contained in:
Sebastian Spaeth 2012-01-07 22:39:59 +01:00
parent 7a5768e471
commit 6dc74c9da5

View File

@ -73,24 +73,32 @@ class LocalStatusSQLiteFolder(LocalStatusFolder):
if version < LocalStatusSQLiteFolder.cur_version: if version < LocalStatusSQLiteFolder.cur_version:
self.upgrade_db(version) self.upgrade_db(version)
def sql_write(self, sql, vars=None): def sql_write(self, sql, vars=None, executemany=False):
"""execute some SQL retrying if the db was locked. """Execute some SQL, retrying if the db was locked.
:param sql: the SQL string passed to execute() :param args: the :param sql: the SQL string passed to execute()
variable values to `sql`. E.g. (1,2) or {uid:1, flags:'T'}. See :param vars: the variable values to `sql`. E.g. (1,2) or {uid:1,
sqlite docs for possibilities. flags:'T'}. See sqlite docs for possibilities.
:param executemany: bool indicating whether we want to
perform conn.executemany() or conn.execute().
:returns: the Cursor() or raises an Exception""" :returns: the Cursor() or raises an Exception"""
success = False success = False
while not success: while not success:
self._dblock.acquire() self._dblock.acquire()
try: try:
if vars is None: if vars is None:
cursor = self.connection.execute(sql) if executemany:
cursor = self.connection.executemany(sql)
else:
cursor = self.connection.execute(sql)
else: else:
cursor = self.connection.execute(sql, vars) if executemany:
cursor = self.connection.executemany(sql, vars)
else:
cursor = self.connection.execute(sql, vars)
success = True success = True
self.connection.commit() self.connection.commit()
except sqlite.OperationalError, e: except sqlite.OperationalError as e:
if e.args[0] == 'cannot commit - no transaction is active': if e.args[0] == 'cannot commit - no transaction is active':
pass pass
elif e.args[0] == 'database is locked': elif e.args[0] == 'database is locked':
@ -231,12 +239,23 @@ class LocalStatusSQLiteFolder(LocalStatusFolder):
flags = ''.join(sorted(flags)) flags = ''.join(sorted(flags))
self.sql_write('UPDATE status SET flags=? WHERE id=?',(flags,uid)) self.sql_write('UPDATE status SET flags=? WHERE id=?',(flags,uid))
def deletemessage(self, uid):
if not uid in self.messagelist:
return
self.sql_write('DELETE FROM status WHERE id=?', (uid, ))
del(self.messagelist[uid])
def deletemessages(self, uidlist): def deletemessages(self, uidlist):
"""Delete list of UIDs from status cache
This function uses sqlites executemany() function which is
much faster than iterating through deletemessage() when we have
many messages to delete."""
# Weed out ones not in self.messagelist # Weed out ones not in self.messagelist
uidlist = [uid for uid in uidlist if uid in self.messagelist] uidlist = [uid for uid in uidlist if uid in self.messagelist]
if not len(uidlist): if not len(uidlist):
return return
# arg2 needs to be an iterable of 1-tuples [(1,),(2,),...]
self.sql_write('DELETE FROM status WHERE id=?', zip(uidlist, ), True)
for uid in uidlist: for uid in uidlist:
del(self.messagelist[uid]) del(self.messagelist[uid])
#TODO: we want a way to do executemany(.., uidlist) to delete all
self.sql_write('DELETE FROM status WHERE id=?', (uid, ))