From 09556d645ea81a53c15ce21383a5b754936cee0a Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Sat, 27 Jul 2013 23:25:13 +0200 Subject: [PATCH] Adapt plain status folder to gmail labels stuff * Implements Status Folder format v2, with a mechanism to upgrade an old statusfolder. * Do not warn about Gmail and GmailMaildir needing sqlite backend anymore. * Clean repository.LocalStatus reusing some code from folder.LocalStatus. * Change field separator in the plaintext file from ':' to '|'. Now the local status stores gmail labels. If they contain field separator character (formerly ':'), they get messed up. The new character '|' is less likely to appear in a label. Signed-off-by: Eygene Ryabinkin --- offlineimap.conf | 3 +- offlineimap/folder/Gmail.py | 2 +- offlineimap/folder/GmailMaildir.py | 2 +- offlineimap/folder/LocalStatus.py | 144 +++++++++++++++++++++++-- offlineimap/repository/Gmail.py | 6 -- offlineimap/repository/GmailMaildir.py | 6 -- offlineimap/repository/LocalStatus.py | 24 +---- 7 files changed, 141 insertions(+), 46 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 7d74df4..3fc20f7 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -272,8 +272,7 @@ remoterepository = RemoteExample #maildir-windows-compatible = no # Specifies if we want to sync GMail lables with the local repository. -# Effective only for GMail IMAP repositories. You should use SQlite -# backend for this to work (see status_backend). +# Effective only for GMail IMAP repositories. # #synclabels = no diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 08185a1..1e315a7 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -284,7 +284,7 @@ class GmailFolder(IMAPFolder): labels = dstfolder.getmessagelabels(uid) statusfolder.savemessagelabels(uid, labels, mtime=mtime) - # either statusfolder is not sqlite or dstfolder is not GmailMaildir. + # dstfolder is not GmailMaildir. except NotImplementedError: return diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py index e94dffe..3f37b02 100644 --- a/offlineimap/folder/GmailMaildir.py +++ b/offlineimap/folder/GmailMaildir.py @@ -191,7 +191,7 @@ class GmailMaildirFolder(MaildirFolder): labels = dstfolder.getmessagelabels(uid) statusfolder.savemessagelabels(uid, labels, mtime=self.getmessagemtime(uid)) - # either statusfolder is not sqlite or dstfolder is not GmailMaildir. + # dstfolder is not GmailMaildir. except NotImplementedError: return diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index c3f24f6..1be80e0 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -19,10 +19,13 @@ from .Base import BaseFolder import os import threading -magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT 1" - class LocalStatusFolder(BaseFolder): + """LocalStatus backend implemented as a plain text file""" + + cur_version = 2 + magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT %d" + def __init__(self, name, repository): self.sep = '.' #needs to be set before super.__init__() super(LocalStatusFolder, self).__init__(name, repository) @@ -76,7 +79,17 @@ class LocalStatusFolder(BaseFolder): file.close() return assert(line == magicline) - for line in file.xreadlines(): + + + def readstatus_v1(self, fp): + """ + Read status folder in format version 1. + + Arguments: + - fp: I/O object that points to the opened database file. + + """ + for line in fp.xreadlines(): line = line.strip() try: uid, flags = line.split(':') @@ -87,17 +100,91 @@ class LocalStatusFolder(BaseFolder): (line, self.filename) self.ui.warn(errstr) raise ValueError(errstr) - self.messagelist[uid] = {'uid': uid, 'flags': flags} + self.messagelist[uid] = {'uid': uid, 'flags': flags, 'mtime': 0, 'labels': set()} + + + def readstatus(self, fp): + """ + Read status file in the current format. + + Arguments: + - fp: I/O object that points to the opened database file. + + """ + for line in fp.xreadlines(): + line = line.strip() + try: + uid, flags, mtime, labels = line.split('|') + uid = long(uid) + flags = set(flags) + mtime = long(mtime) + labels = set([lb.strip() for lb in labels.split(',') if len(lb.strip()) > 0]) + except ValueError as e: + errstr = "Corrupt line '%s' in cache file '%s'" % \ + (line, self.filename) + self.ui.warn(errstr) + raise ValueError(errstr) + self.messagelist[uid] = {'uid': uid, 'flags': flags, 'mtime': mtime, 'labels': labels} + + + def cachemessagelist(self): + if self.isnewfolder(): + self.messagelist = {} + return + + # loop as many times as version, and update format + for i in range(1, self.cur_version+1): + file = open(self.filename, "rt") + self.messagelist = {} + line = file.readline().strip() + + # convert from format v1 + if line == (self.magicline % 1): + self.ui._msg('Upgrading LocalStatus cache from version 1 to version 2 for %s:%s' %\ + (self.repository, self)) + self.readstatus_v1(file) + file.close() + self.save() + + # NOTE: Add other format transitions here in the future. + # elif line == (self.magicline % 2): + # self.ui._msg('Upgrading LocalStatus cache from version 2 to version 3 for %s:%s' %\ + # (self.repository, self)) + # self.readstatus_v2(file) + # file.close() + # file.save() + + # format is up to date. break + elif line == (self.magicline % self.cur_version): + break + + # something is wrong + else: + errstr = "Unrecognized cache magicline in '%s'" % self.filename + self.ui.warn(errstr) + raise ValueError(errstr) + + if not line: + # The status file is empty - should not have happened, + # but somehow did. + errstr = "Cache file '%s' is empty. Closing..." % self.filename + self.ui.warn(errstr) + file.close() + return + + assert(line == (self.magicline % self.cur_version)) + self.readstatus(file) file.close() + def save(self): with self.savelock: file = open(self.filename + ".tmp", "wt") - file.write(magicline + "\n") + file.write((self.magicline % self.cur_version) + "\n") for msg in self.messagelist.values(): - flags = msg['flags'] - flags = ''.join(sorted(flags)) - file.write("%s:%s\n" % (msg['uid'], flags)) + flags = ''.join(sorted(msg['flags'])) + labels = ', '.join(sorted(msg['labels'])) + file.write("%s|%s|%d|%s\n" % (msg['uid'], flags, msg['mtime'], labels)) file.flush() if self.doautosave: os.fsync(file.fileno()) @@ -114,7 +201,7 @@ class LocalStatusFolder(BaseFolder): return self.messagelist # Interface from BaseFolder - def savemessage(self, uid, content, flags, rtime): + def savemessage(self, uid, content, flags, rtime, mtime=0, labels=set()): """Writes a new message, with the specified uid. See folder/Base for detail. Note that savemessage() does not @@ -124,11 +211,11 @@ class LocalStatusFolder(BaseFolder): # We cannot assign a uid. return uid - if uid in self.messagelist: # already have it + if self.uidexists(uid): # already have it self.savemessageflags(uid, flags) return uid - self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} + self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime, 'mtime': mtime, 'labels': labels} self.save() return uid @@ -145,6 +232,41 @@ class LocalStatusFolder(BaseFolder): self.messagelist[uid]['flags'] = flags self.save() + + def savemessagelabels(self, uid, labels, mtime=None): + self.messagelist[uid]['labels'] = labels + if mtime: self.messagelist[uid]['mtime'] = mtime + self.save() + + def savemessageslabelsbulk(self, labels): + """Saves labels from a dictionary in a single database operation.""" + for uid, lb in labels.items(): + self.messagelist[uid]['labels'] = lb + self.save() + + def addmessageslabels(self, uids, labels): + for uid in uids: + self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] | labels + self.save() + + def deletemessageslabels(self, uids, labels): + for uid in uids: + self.messagelist[uid]['labels'] = self.messagelist[uid]['labels'] - labels + self.save() + + def getmessagelabels(self, uid): + return self.messagelist[uid]['labels'] + + def savemessagesmtimebulk(self, mtimes): + """Saves mtimes from the mtimes dictionary in a single database operation.""" + for uid, mt in mtimes.items(): + self.messagelist[uid]['mtime'] = mt + self.save() + + def getmessagemtime(self, uid): + return self.messagelist[uid]['mtime'] + + # Interface from BaseFolder def deletemessage(self, uid): self.deletemessages([uid]) diff --git a/offlineimap/repository/Gmail.py b/offlineimap/repository/Gmail.py index 61d4486..2e23e62 100644 --- a/offlineimap/repository/Gmail.py +++ b/offlineimap/repository/Gmail.py @@ -36,12 +36,6 @@ class GmailRepository(IMAPRepository): 'ssl', 'yes') IMAPRepository.__init__(self, reposname, account) - if self.account.getconfboolean('synclabels', 0) and \ - self.account.getconf('status_backend', 'plain') != 'sqlite': - raise OfflineImapError("The Gmail repository needs the sqlite backend to sync labels.\n" - "To enable it add 'status_backend = sqlite' in the account section", - OfflineImapError.ERROR.REPO) - def gethost(self): """Return the server name to connect to. diff --git a/offlineimap/repository/GmailMaildir.py b/offlineimap/repository/GmailMaildir.py index 62f9f83..9072b7c 100644 --- a/offlineimap/repository/GmailMaildir.py +++ b/offlineimap/repository/GmailMaildir.py @@ -25,12 +25,6 @@ class GmailMaildirRepository(MaildirRepository): """Initialize a MaildirRepository object. Takes a path name to the directory holding all the Maildir directories.""" super(GmailMaildirRepository, self).__init__(reposname, account) - if self.account.getconfboolean('synclabels', 0) and \ - self.account.getconf('status_backend', 'plain') != 'sqlite': - raise OfflineImapError("The GmailMaildir repository needs the sqlite backend to sync labels.\n" - "To enable it add 'status_backend = sqlite' in the account section", - OfflineImapError.ERROR.REPO) - def getfoldertype(self): diff --git a/offlineimap/repository/LocalStatus.py b/offlineimap/repository/LocalStatus.py index bb9ada4..b75d44a 100644 --- a/offlineimap/repository/LocalStatus.py +++ b/offlineimap/repository/LocalStatus.py @@ -16,7 +16,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -from offlineimap.folder.LocalStatus import LocalStatusFolder, magicline +from offlineimap.folder.LocalStatus import LocalStatusFolder from offlineimap.folder.LocalStatusSQLite import LocalStatusSQLiteFolder from offlineimap.repository.Base import BaseRepository import os @@ -49,19 +49,6 @@ class LocalStatusRepository(BaseRepository): def getsep(self): return '.' - def getfolderfilename(self, foldername): - """Return the full path of the status file - - This mimics the path that Folder().getfolderbasename() would return""" - if not foldername: - basename = '.' - else: #avoid directory hierarchies and file names such as '/' - basename = foldername.replace('/', '.') - # replace with literal 'dot' if final path name is '.' as '.' is - # an invalid file name. - basename = re.sub('(^|\/)\.$','\\1dot', basename) - return os.path.join(self.root, basename) - def makefolder(self, foldername): """Create a LocalStatus Folder @@ -73,11 +60,10 @@ class LocalStatusRepository(BaseRepository): if self.account.dryrun: return # bail out in dry-run mode - filename = self.getfolderfilename(foldername) - file = open(filename + ".tmp", "wt") - file.write(magicline + '\n') - file.close() - os.rename(filename + ".tmp", filename) + # Create an empty StatusFolder + folder = self.LocalStatusFolderClass(foldername, self) + folder.save() + # Invalidate the cache. self._folders = {}