From 1b954c3b4c8c623c5878a8387e125a2aef0873ed Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Sun, 28 Jul 2013 14:03:46 +0200 Subject: [PATCH] Add ability to migrate status data across backends If when we request a LocalStatus folder, the folder has to be created, we look whether the other backend has data, and if it does we migrate it to the new backend. The old backend data is left untouched, so that if you change back say from sqlite to plaintext, the older data is still there. That should not lead to data loss, only a slower sync while the status folder gets updated. Signed-off-by: Eygene Ryabinkin --- Changelog.rst | 2 + offlineimap/folder/LocalStatusSQLite.py | 28 ---------- offlineimap/repository/LocalStatus.py | 73 ++++++++++++++++++------- 3 files changed, 54 insertions(+), 49 deletions(-) diff --git a/Changelog.rst b/Changelog.rst index d2148e6..4fd26a1 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -14,6 +14,8 @@ OfflineIMAP v6.5.6 (YYYY-MM-DD) to IMAP server (Abdó Roig-Maranges) * Allow to sync GMail labels and implement GmailMaildir repository that adds mechanics to change message labels (Abdó Roig-Maranges) +* Allow to migrate status data across differend backends + (Abdó Roig-Maranges) OfflineIMAP v6.5.5 (2013-10-07) diff --git a/offlineimap/folder/LocalStatusSQLite.py b/offlineimap/folder/LocalStatusSQLite.py index 010e81b..f8921f1 100644 --- a/offlineimap/folder/LocalStatusSQLite.py +++ b/offlineimap/folder/LocalStatusSQLite.py @@ -144,34 +144,6 @@ class LocalStatusSQLiteFolder(BaseFolder): self.connection = sqlite.connect(self.filename, check_same_thread = False) - if from_ver == 0: - # from_ver==0: no db existent: plain text migration? - self.__create_db() - # below was derived from repository.getfolderfilename() logic - plaintextfilename = os.path.join( - self.repository.account.getaccountmeta(), - 'LocalStatus', - self.getfolderbasename()) - # MIGRATE from plaintext if needed - # TODO: adopt for plain-text v2 - if os.path.exists(plaintextfilename): - self.ui._msg('Migrating LocalStatus cache from plain text ' - 'to sqlite database for %s:%s' %\ - (self.repository, self)) - file = open(plaintextfilename, "rt") - line = file.readline().strip() - data = [] - for line in file.xreadlines(): - uid, flags = line.strip().split(':') - uid = long(uid) - flags = ''.join(sorted(flags)) - data.append((uid,flags)) - self.connection.executemany('INSERT INTO status (id,flags) VALUES (?,?)', - data) - self.connection.commit() - file.close() - os.rename(plaintextfilename, plaintextfilename + ".old") - # Upgrade from database version 1 to version 2 # This change adds labels and mtime columns, to be used by Gmail IMAP and Maildir folders. if from_ver <= 1: diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index b75d44a..3bd97ce 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -25,20 +25,21 @@ import re class LocalStatusRepository(BaseRepository): def __init__(self, reposname, account): BaseRepository.__init__(self, reposname, account) - # Root directory in which the LocalStatus folders reside - self.root = os.path.join(account.getaccountmeta(), 'LocalStatus') - # statusbackend can be 'plain' or 'sqlite' - backend = self.account.getconf('status_backend', 'plain') - if backend == 'sqlite': - self._backend = 'sqlite' - self.LocalStatusFolderClass = LocalStatusSQLiteFolder - self.root += '-sqlite' - elif backend == 'plain': - self._backend = 'plain' - self.LocalStatusFolderClass = LocalStatusFolder - else: - raise SyntaxWarning("Unknown status_backend '%s' for account '%s'" \ - % (backend, account.name)) + + # class and root for all backends + self.backends = {} + self.backends['sqlite'] = { + 'class': LocalStatusSQLiteFolder, + 'root': os.path.join(account.getaccountmeta(), 'LocalStatus-sqlite') + } + + self.backends['plain'] = { + 'class': LocalStatusFolder, + 'root': os.path.join(account.getaccountmeta(), 'LocalStatus') + } + + # Set class and root for the configured backend + self.setup_backend(self.account.getconf('status_backend', 'plain')) if not os.path.exists(self.root): os.mkdir(self.root, 0o700) @@ -46,16 +47,41 @@ class LocalStatusRepository(BaseRepository): # self._folders is a dict of name:LocalStatusFolders() self._folders = {} + def setup_backend(self, backend): + if backend in self.backends.keys(): + self._backend = backend + self.root = self.backends[backend]['root'] + self.LocalStatusFolderClass = self.backends[backend]['class'] + + else: + raise SyntaxWarning("Unknown status_backend '%s' for account '%s'" \ + % (backend, account.name)) + + def import_other_backend(self, folder): + for bk, dic in self.backends.items(): + # skip folder's own type + if dic['class'] == type(folder): + continue + + repobk = LocalStatusRepository(self.name, self.account) + repobk.setup_backend(bk) # fake the backend + folderbk = dic['class'](folder.name, repobk) + + # if backend contains data, import it to folder. + if not folderbk.isnewfolder(): + self.ui._msg('Migrating LocalStatus cache from %s to %s ' % (bk, self._backend) + \ + 'status folder for %s:%s' % (self.name, folder.name)) + + folderbk.cachemessagelist() + folder.messagelist = folderbk.messagelist + folder.saveall() + break + def getsep(self): return '.' def makefolder(self, foldername): - """Create a LocalStatus Folder - - Empty Folder for plain backend. NoOp for sqlite backend as those - are created on demand.""" - if self._backend == 'sqlite': - return # noop for sqlite which creates on-demand + """Create a LocalStatus Folder""" if self.account.dryrun: return # bail out in dry-run mode @@ -65,7 +91,7 @@ class LocalStatusRepository(BaseRepository): folder.save() # Invalidate the cache. - self._folders = {} + self.forgetfolders() def getfolder(self, foldername): """Return the Folder() object for a foldername""" @@ -73,6 +99,11 @@ class LocalStatusRepository(BaseRepository): return self._folders[foldername] folder = self.LocalStatusFolderClass(foldername, self) + + # if folder is empty, try to import data from an other backend + if folder.isnewfolder(): + self.import_other_backend(folder) + self._folders[foldername] = folder return folder