From fc4c7549d6d03415edc2537b29c83fd6baea1bc6 Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Thu, 20 Nov 2014 14:03:15 +0100 Subject: [PATCH 1/3] Do not ignore gmail labels if header appears multiple times There should be just one header storing gmail labels, but due to a bug, multiple X-Keywords (or equivalent) headers may be found on the local messages. Now we, when extracting the labels from a message, we read all label headers, instead of just the first one. This has the consequence that some old labels stored locally in a second X-Keywords (or third...) header, which effectively was rendered invisible to offlineimap until now, may pop back up again and be pushed to gmail. No labels will be removed by the changes in this commit, though. Signed-off-by: Abdo Roig-Maranges --- offlineimap/folder/Base.py | 25 +++++++++++++++++++++++-- offlineimap/folder/Gmail.py | 5 +++-- offlineimap/folder/GmailMaildir.py | 19 +++++++++++-------- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index b56cd1b..4b30d8c 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -512,8 +512,8 @@ class BaseFolder(object): def getmessageheader(self, content, name): """ - Searches for the given header and returns its value. - Header name is case-insensitive. + Searches for the first occurence of the given header and returns + its value. Header name is case-insensitive. Arguments: - contents: message itself @@ -535,6 +535,27 @@ class BaseFolder(object): return None + def getmessageheaderlist(self, content, name): + """ + Searches for the given header and returns a list of values for + that header. + + Arguments: + - contents: message itself + - name: name of the header to be searched + + Returns: list of header values or emptylist if no such header was found + + """ + self.ui.debug('', 'getmessageheaderlist: called to get %s' % name) + eoh = self.__find_eoh(content) + self.ui.debug('', 'getmessageheaderlist: eoh = %d' % eoh) + headers = content[0:eoh] + self.ui.debug('', 'getmessageheaderlist: headers = %s' % repr(headers)) + + return re.findall('^%s:(.*)$' % name, headers, flags = re.MULTILINE | re.IGNORECASE) + + def deletemessageheaders(self, content, header_list): """ Deletes headers in the given list from the message content. diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index e3ef92a..2a8afd2 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -189,8 +189,9 @@ class GmailFolder(IMAPFolder): if not self.synclabels: return super(GmailFolder, self).savemessage(uid, content, flags, rtime) - labels = imaputil.labels_from_header(self.labelsheader, - self.getmessageheader(content, self.labelsheader)) + labels = set() + for hstr in self.getmessageheaderlist(content, self.labelsheader): + labels.update(imaputil.labels_from_header(self.labelsheader, hstr)) ret = super(GmailFolder, self).savemessage(uid, content, flags, rtime) self.savemessagelabels(ret, labels) diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py index 2cab213..7a67c87 100644 --- a/offlineimap/folder/GmailMaildir.py +++ b/offlineimap/folder/GmailMaildir.py @@ -87,9 +87,10 @@ class GmailMaildirFolder(MaildirFolder): content = file.read() file.close() - self.messagelist[uid]['labels'] = \ - imaputil.labels_from_header(self.labelsheader, - self.getmessageheader(content, self.labelsheader)) + self.messagelist[uid]['labels'] = set() + for hstr in self.getmessageheaderlist(content, self.labelsheader): + self.messagelist[uid]['labels'].update( + imaputil.labels_from_header(self.labelsheader, hstr)) self.messagelist[uid]['labels_cached'] = True return self.messagelist[uid]['labels'] @@ -111,8 +112,10 @@ class GmailMaildirFolder(MaildirFolder): if not self.synclabels: return super(GmailMaildirFolder, self).savemessage(uid, content, flags, rtime) - labels = imaputil.labels_from_header(self.labelsheader, - self.getmessageheader(content, self.labelsheader)) + labels = set() + for hstr in self.getmessageheaderlist(content, self.labelsheader): + labels.update(imaputil.labels_from_header(self.labelsheader, hstr)) + ret = super(GmailMaildirFolder, self).savemessage(uid, content, flags, rtime) # Update the mtime and labels @@ -135,9 +138,9 @@ class GmailMaildirFolder(MaildirFolder): content = file.read() file.close() - oldlabels = imaputil.labels_from_header(self.labelsheader, - self.getmessageheader(content, self.labelsheader)) - + oldlabels = set() + for hstr in self.getmessageheaderlist(content, self.labelsheader): + oldlabels.update(imaputil.labels_from_header(self.labelsheader, hstr)) labels = labels - ignorelabels ignoredlabels = oldlabels & ignorelabels From 2a5ef8c2ef2d6d64ef9ab8c6216ef68d6f07937c Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Thu, 20 Nov 2014 14:16:48 +0100 Subject: [PATCH 2/3] Delete gmail labels header before adding a new one This fixes a bug in which a message ended up with multiple gmail labels header (X-Keywords or so). Fix fix it by removing all labels headers before adding the updated one. Signed-off-by: Abdo Roig-Maranges --- offlineimap/folder/Gmail.py | 4 ++++ offlineimap/folder/GmailMaildir.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index 2a8afd2..69f0075 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -93,6 +93,10 @@ class GmailFolder(IMAPFolder): labels = set() labels = labels - self.ignorelabels labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels)) + + # First remove old label headers that may be in the message content retrieved + # from gmail Then add a labels header with current gmail labels. + body = self.deletemessageheaders(body, self.labelsheader) body = self.addmessageheader(body, '\n', self.labelsheader, labels_str) if len(body)>200: diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py index 7a67c87..3b127d2 100644 --- a/offlineimap/folder/GmailMaildir.py +++ b/offlineimap/folder/GmailMaildir.py @@ -153,7 +153,11 @@ class GmailMaildirFolder(MaildirFolder): # Change labels into content labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels | ignoredlabels)) + + # First remove old labels header, and then add the new one + content = self.deletemessageheaders(content, self.labelsheader) content = self.addmessageheader(content, '\n', self.labelsheader, labels_str) + rtime = self.messagelist[uid].get('rtime', None) # write file with new labels to a unique file in tmp From e70607e3e33701cc3b7fcc4cc952f57097a0ac12 Mon Sep 17 00:00:00 2001 From: Abdo Roig-Maranges Date: Thu, 20 Nov 2014 14:18:35 +0100 Subject: [PATCH 3/3] Warn about a tricky piece of code in addmessageheader As requested by Nicholas Sebrecht. Signed-off-by: Abdo Roig-Maranges --- offlineimap/folder/Base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 4b30d8c..106c390 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -401,6 +401,9 @@ class BaseFolder(object): """ Adds new header to the provided message. + WARNING: This function is a bit tricky, and modifying it in the wrong way, + may easily lead to data-loss. + Arguments: - content: message content, headers and body as a single string - linebreak: string that carries line ending