From 6dc74c9da5016a8fe7b30bcc438ae8b700084824 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sat, 7 Jan 2012 22:39:59 +0100 Subject: [PATCH] Improve delete performance with SQLITE backend When deleting many (eg 2000) mails using the SQLITE backend, this takes a long time durig which OfflineImap can not be aborted via CTRL-C. Thinking it had frozen permanently, I killed it hard, leaving a corrupted db journal (which leads to awkwards complaints by OLI on subsequent starts!). That shows that delete performance is critical and needs improvement. We were iterating through the list of messages to delete and deleted them one-by-one execute()'ing a new SQL Query for each message. This patch improves the situation by allowing us to use executemany(), which is -despite still being one SQL query per message- much faster. This is because rather than performing a commit() after each mail, we now do only one commit() after all mails have been deleted. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/LocalStatusSQLite.py | 39 ++++++++++++++++++------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index 08af807..6bfa667 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -73,24 +73,32 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): if version < LocalStatusSQLiteFolder.cur_version: self.upgrade_db(version) - def sql_write(self, sql, vars=None): - """execute some SQL retrying if the db was locked. + def sql_write(self, sql, vars=None, executemany=False): + """Execute some SQL, retrying if the db was locked. - :param sql: the SQL string passed to execute() :param args: the - variable values to `sql`. E.g. (1,2) or {uid:1, flags:'T'}. See - sqlite docs for possibilities. + :param sql: the SQL string passed to execute() + :param vars: the variable values to `sql`. E.g. (1,2) or {uid:1, + flags:'T'}. See sqlite docs for possibilities. + :param executemany: bool indicating whether we want to + perform conn.executemany() or conn.execute(). :returns: the Cursor() or raises an Exception""" success = False while not success: self._dblock.acquire() try: if vars is None: - cursor = self.connection.execute(sql) + if executemany: + cursor = self.connection.executemany(sql) + else: + cursor = self.connection.execute(sql) else: - cursor = self.connection.execute(sql, vars) + if executemany: + cursor = self.connection.executemany(sql, vars) + else: + cursor = self.connection.execute(sql, vars) success = True self.connection.commit() - except sqlite.OperationalError, e: + except sqlite.OperationalError as e: if e.args[0] == 'cannot commit - no transaction is active': pass elif e.args[0] == 'database is locked': @@ -231,12 +239,23 @@ class LocalStatusSQLiteFolder(LocalStatusFolder): flags = ''.join(sorted(flags)) self.sql_write('UPDATE status SET flags=? WHERE id=?',(flags,uid)) + def deletemessage(self, uid): + if not uid in self.messagelist: + return + self.sql_write('DELETE FROM status WHERE id=?', (uid, )) + del(self.messagelist[uid]) + def deletemessages(self, uidlist): + """Delete list of UIDs from status cache + + This function uses sqlites executemany() function which is + much faster than iterating through deletemessage() when we have + many messages to delete.""" # Weed out ones not in self.messagelist uidlist = [uid for uid in uidlist if uid in self.messagelist] if not len(uidlist): return + # arg2 needs to be an iterable of 1-tuples [(1,),(2,),...] + self.sql_write('DELETE FROM status WHERE id=?', zip(uidlist, ), True) for uid in uidlist: del(self.messagelist[uid]) - #TODO: we want a way to do executemany(.., uidlist) to delete all - self.sql_write('DELETE FROM status WHERE id=?', (uid, ))