sqlite: close the database when no more threads need access

It's required to have more than one connection to the database for the
maxconnections configuration option to work with threads. However,
connection.close() is closing all the connections. Only close the connection
when no more thread need it.

Backported-from: 856b74407bd7f634cae5a8c2d9b84e13d14c12d2
Github-fix: https://github.com/OfflineIMAP/offlineimap/issues/350
Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>

offlineimap/folder/LocalStatusSQLite.py

@@ -42,6 +42,9 @@ class LocalStatusSQLiteFolder(BaseFolder):
 
     # Current version of our db format.
     cur_version = 2
+    # Keep track on how many threads need access to the database.
+    threads_open_count = 0
+    threads_open_lock = Lock()
 
     def __init__(self, name, repository):
         self.sep = '.' # Needs to be set before super.__init__()
@@ -64,36 +67,39 @@ class LocalStatusSQLiteFolder(BaseFolder):
         self.connection = None
 
     def openfiles(self):
-        # Try to establish connection, no need for threadsafety in __init__.
+        # Protect the creation/upgrade of database accross threads.
-        try:
+        with LocalStatusSQLiteFolder.threads_open_lock:
-            self.connection = sqlite.connect(self.filename, check_same_thread=False)
+            # Try to establish connection, no need for threadsafety in __init__.
-        except NameError:
+            try:
-            # sqlite import had failed.
+                self.connection = sqlite.connect(self.filename, check_same_thread=False)
-            raise UserWarning("SQLite backend chosen, but cannot connect "
+                LocalStatusSQLiteFolder.threads_open_count += 1
-                "with available bindings to '%s'. Is the sqlite3 package "
+            except NameError:
-                "installed?."% self.filename), None, exc_info()[2]
+                # sqlite import had failed.
-        except sqlite.OperationalError as e:
+                raise UserWarning("SQLite backend chosen, but cannot connect "
-            # Operation had failed.
+                    "with available bindings to '%s'. Is the sqlite3 package "
-            raise UserWarning("cannot open database file '%s': %s.\nYou might "
+                    "installed?."% self.filename), None, exc_info()[2]
-                "want to check the rights to that file and if it cleanly opens "
+            except sqlite.OperationalError as e:
-                "with the 'sqlite<3>' command."%
+                # Operation had failed.
-                (self.filename, e)), None, exc_info()[2]
+                raise UserWarning("cannot open database file '%s': %s.\nYou might "
+                    "want to check the rights to that file and if it cleanly opens "
+                    "with the 'sqlite<3>' command."%
+                    (self.filename, e)), None, exc_info()[2]
 
-        # Make sure sqlite is in multithreading SERIALIZE mode.
+            # Make sure sqlite is in multithreading SERIALIZE mode.
-        assert sqlite.threadsafety == 1, 'Your sqlite is not multithreading safe.'
+            assert sqlite.threadsafety == 1, 'Your sqlite is not multithreading safe.'
 
-        # Test if db version is current enough and if db is readable.
+            # Test if db version is current enough and if db is readable.
-        try:
+            try:
-            cursor = self.connection.execute(
+                cursor = self.connection.execute(
-                "SELECT value from metadata WHERE key='db_version'")
+                    "SELECT value from metadata WHERE key='db_version'")
-        except sqlite.DatabaseError:
+            except sqlite.DatabaseError:
-            # db file missing or corrupt, recreate it.
+                # db file missing or corrupt, recreate it.
-            self.__create_db()
+                self.__create_db()
-        else:
+            else:
-            # Fetch db version and upgrade if needed.
+                # Fetch db version and upgrade if needed.
-            version = int(cursor.fetchone()[0])
+                version = int(cursor.fetchone()[0])
-            if version < LocalStatusSQLiteFolder.cur_version:
+                if version < LocalStatusSQLiteFolder.cur_version:
-                self.__upgrade_db(version)
+                    self.__upgrade_db(version)
 
 
     def storesmessages(self):
@@ -155,8 +161,8 @@ class LocalStatusSQLiteFolder(BaseFolder):
     def __upgrade_db(self, from_ver):
         """Upgrade the sqlite format from version 'from_ver' to current"""
 
-        if hasattr(self, 'connection'):
+        if self.connection is not None:
-            self.connection.close() #close old connections first
+            self.connection.close() # Close old connections first.
         self.connection = sqlite.connect(self.filename,
                                          check_same_thread = False)
 
@@ -181,8 +187,8 @@ class LocalStatusSQLiteFolder(BaseFolder):
 
         self.connection must point to the opened and valid SQlite
         database connection."""
-        self.ui._msg('Creating new Local Status db for %s:%s' \
+        self.ui._msg('Creating new Local Status db for %s:%s'%
-                         % (self.repository, self))
+                     (self.repository, self))
         self.connection.executescript("""
         CREATE TABLE metadata (key VARCHAR(50) PRIMARY KEY, value VARCHAR(128));
         INSERT INTO metadata VALUES('db_version', '2');
@@ -225,10 +231,13 @@ class LocalStatusSQLiteFolder(BaseFolder):
             self.messagelist[uid]['mtime'] = row[2]
 
     def closefiles(self):
-        try:
+        with LocalStatusSQLiteFolder.threads_open_lock:
-            self.connection.close()
+            LocalStatusSQLiteFolder.threads_open_count -= 1
-        except:
+            if self.threads_open_count < 1:
-            pass
+                try:
+                    self.connection.close()
+                except:
+                    pass
 
     def dropmessagelistcache(self):
         self.messagelist = {}