From 677afb8d8f4b3d41e3842150131a5638fc60c805 Mon Sep 17 00:00:00 2001 From: Nicolas Sebrecht Date: Fri, 22 Jul 2016 00:08:45 +0200 Subject: [PATCH] sqlite: close the database when no more threads need access It's required to have more than one connection to the database for the maxconnections configuration option to work with threads. However, connection.close() is closing all the connections. Only close the connection when no more thread need it. Backported-from: 856b74407bd7f634cae5a8c2d9b84e13d14c12d2 Github-fix: https://github.com/OfflineIMAP/offlineimap/issues/350 Signed-off-by: Nicolas Sebrecht --- offlineimap/folder/LocalStatusSQLite.py | 81 ++++++++++++++----------- 1 file changed, 45 insertions(+), 36 deletions(-) diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index 79c6e99..8d3e9ee 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -42,6 +42,9 @@ class LocalStatusSQLiteFolder(BaseFolder): # Current version of our db format. cur_version = 2 + # Keep track on how many threads need access to the database. + threads_open_count = 0 + threads_open_lock = Lock() def __init__(self, name, repository): self.sep = '.' # Needs to be set before super.__init__() @@ -64,36 +67,39 @@ class LocalStatusSQLiteFolder(BaseFolder): self.connection = None def openfiles(self): - # Try to establish connection, no need for threadsafety in __init__. - try: - self.connection = sqlite.connect(self.filename, check_same_thread=False) - except NameError: - # sqlite import had failed. - raise UserWarning("SQLite backend chosen, but cannot connect " - "with available bindings to '%s'. Is the sqlite3 package " - "installed?."% self.filename), None, exc_info()[2] - except sqlite.OperationalError as e: - # Operation had failed. - raise UserWarning("cannot open database file '%s': %s.\nYou might " - "want to check the rights to that file and if it cleanly opens " - "with the 'sqlite<3>' command."% - (self.filename, e)), None, exc_info()[2] + # Protect the creation/upgrade of database accross threads. + with LocalStatusSQLiteFolder.threads_open_lock: + # Try to establish connection, no need for threadsafety in __init__. + try: + self.connection = sqlite.connect(self.filename, check_same_thread=False) + LocalStatusSQLiteFolder.threads_open_count += 1 + except NameError: + # sqlite import had failed. + raise UserWarning("SQLite backend chosen, but cannot connect " + "with available bindings to '%s'. Is the sqlite3 package " + "installed?."% self.filename), None, exc_info()[2] + except sqlite.OperationalError as e: + # Operation had failed. + raise UserWarning("cannot open database file '%s': %s.\nYou might " + "want to check the rights to that file and if it cleanly opens " + "with the 'sqlite<3>' command."% + (self.filename, e)), None, exc_info()[2] - # Make sure sqlite is in multithreading SERIALIZE mode. - assert sqlite.threadsafety == 1, 'Your sqlite is not multithreading safe.' + # Make sure sqlite is in multithreading SERIALIZE mode. + assert sqlite.threadsafety == 1, 'Your sqlite is not multithreading safe.' - # Test if db version is current enough and if db is readable. - try: - cursor = self.connection.execute( - "SELECT value from metadata WHERE key='db_version'") - except sqlite.DatabaseError: - # db file missing or corrupt, recreate it. - self.__create_db() - else: - # Fetch db version and upgrade if needed. - version = int(cursor.fetchone()[0]) - if version < LocalStatusSQLiteFolder.cur_version: - self.__upgrade_db(version) + # Test if db version is current enough and if db is readable. + try: + cursor = self.connection.execute( + "SELECT value from metadata WHERE key='db_version'") + except sqlite.DatabaseError: + # db file missing or corrupt, recreate it. + self.__create_db() + else: + # Fetch db version and upgrade if needed. + version = int(cursor.fetchone()[0]) + if version < LocalStatusSQLiteFolder.cur_version: + self.__upgrade_db(version) def storesmessages(self): @@ -155,8 +161,8 @@ class LocalStatusSQLiteFolder(BaseFolder): def __upgrade_db(self, from_ver): """Upgrade the sqlite format from version 'from_ver' to current""" - if hasattr(self, 'connection'): - self.connection.close() #close old connections first + if self.connection is not None: + self.connection.close() # Close old connections first. self.connection = sqlite.connect(self.filename, check_same_thread = False) @@ -181,8 +187,8 @@ class LocalStatusSQLiteFolder(BaseFolder): self.connection must point to the opened and valid SQlite database connection.""" - self.ui._msg('Creating new Local Status db for %s:%s' \ - % (self.repository, self)) + self.ui._msg('Creating new Local Status db for %s:%s'% + (self.repository, self)) self.connection.executescript(""" CREATE TABLE metadata (key VARCHAR(50) PRIMARY KEY, value VARCHAR(128)); INSERT INTO metadata VALUES('db_version', '2'); @@ -225,10 +231,13 @@ class LocalStatusSQLiteFolder(BaseFolder): self.messagelist[uid]['mtime'] = row[2] def closefiles(self): - try: - self.connection.close() - except: - pass + with LocalStatusSQLiteFolder.threads_open_lock: + LocalStatusSQLiteFolder.threads_open_count -= 1 + if self.threads_open_count < 1: + try: + self.connection.close() + except: + pass def dropmessagelistcache(self): self.messagelist = {}