Improve delete performance with SQLITE backend

When deleting many (eg 2000) mails using the SQLITE backend, this takes
a long time durig which OfflineImap can not be aborted via
CTRL-C. Thinking it had frozen permanently, I killed it hard, leaving a
corrupted db journal (which leads to awkwards complaints by OLI on
subsequent starts!). That shows that delete performance is critical and
needs improvement.

We were iterating through the list of messages to delete and deleted
them one-by-one execute()'ing a new SQL Query for each message. This
patch improves the situation by allowing us to use executemany(), which
is -despite still being one SQL query per message- much faster. This is
because rather than performing a commit() after each mail, we now do
only one commit() after all mails have been deleted.

Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de>

offlineimap/folder/LocalStatusSQLite.py

@@ -73,24 +73,32 @@ class LocalStatusSQLiteFolder(LocalStatusFolder):
             if version < LocalStatusSQLiteFolder.cur_version:
                 self.upgrade_db(version)
 
-    def sql_write(self, sql, vars=None):
+    def sql_write(self, sql, vars=None, executemany=False):
-        """execute some SQL retrying if the db was locked.
+        """Execute some SQL, retrying if the db was locked.
 
-        :param sql: the SQL string passed to execute() :param args: the
+        :param sql: the SQL string passed to execute()
-            variable values to `sql`. E.g. (1,2) or {uid:1, flags:'T'}. See
+        :param vars: the variable values to `sql`. E.g. (1,2) or {uid:1,
-            sqlite docs for possibilities.
+            flags:'T'}. See sqlite docs for possibilities.
+        :param executemany: bool indicating whether we want to
+            perform conn.executemany() or conn.execute().
         :returns: the Cursor() or raises an Exception"""
         success = False
         while not success:
             self._dblock.acquire()
             try:
                 if vars is None:
-                    cursor = self.connection.execute(sql)
+                    if executemany:
+                        cursor = self.connection.executemany(sql)
+                    else:
+                        cursor = self.connection.execute(sql)
                 else:
-                    cursor = self.connection.execute(sql, vars)
+                    if executemany:
+                        cursor = self.connection.executemany(sql, vars)
+                    else:
+                        cursor = self.connection.execute(sql, vars)
                 success = True
                 self.connection.commit()
-            except sqlite.OperationalError, e:
+            except sqlite.OperationalError as e:
                 if e.args[0] == 'cannot commit - no transaction is active':
                     pass
                 elif e.args[0] == 'database is locked':
@@ -231,12 +239,23 @@ class LocalStatusSQLiteFolder(LocalStatusFolder):
         flags = ''.join(sorted(flags))
         self.sql_write('UPDATE status SET flags=? WHERE id=?',(flags,uid))
 
+    def deletemessage(self, uid):
+        if not uid in self.messagelist:
+            return
+        self.sql_write('DELETE FROM status WHERE id=?', (uid, ))
+        del(self.messagelist[uid])
+
     def deletemessages(self, uidlist):
+        """Delete list of UIDs from status cache
+
+        This function uses sqlites executemany() function which is
+        much faster than iterating through deletemessage() when we have
+        many messages to delete."""
         # Weed out ones not in self.messagelist
         uidlist = [uid for uid in uidlist if uid in self.messagelist]
         if not len(uidlist):
             return
+        # arg2 needs to be an iterable of 1-tuples [(1,),(2,),...]
+        self.sql_write('DELETE FROM status WHERE id=?', zip(uidlist, ), True)
         for uid in uidlist:
             del(self.messagelist[uid])
-            #TODO: we want a way to do executemany(.., uidlist) to delete all
-            self.sql_write('DELETE FROM status WHERE id=?', (uid, ))