diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 0ddb284..7a1a04e 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -60,9 +60,9 @@ class BaseFolder: policy.default.clone(cte_type='8bit', utf8=True, refold_source='none', linesep='\r\n'), } # Parsers - self.parse = { - '8bit': BytesParser(policy=p1), - } + self.parser = {} + for key in self.policy: + self.parser[key] = BytesParser(policy=self.policy[key]) # Save original name for folderfilter operations. self.ffilter_name = name # Top level dir name is always ''. diff --git a/offlineimap/folder/Gmail.py b/offlineimap/folder/Gmail.py index f78f9c3..d047cf2 100644 --- a/offlineimap/folder/Gmail.py +++ b/offlineimap/folder/Gmail.py @@ -69,10 +69,9 @@ class GmailFolder(IMAPFolder): data = self._fetch_from_imap(str(uid), self.retrycount) # data looks now e.g. - # ['320 (X-GM-LABELS (...) UID 17061 BODY[] {2565}','msgbody....'] + # ['320 (X-GM-LABELS (...) UID 17061 BODY[] {2565}',] # we only asked for one message, and that msg is in data[1]. - # msbody is in [1]. - body = data[1].replace("\r\n", "\n") + msg = data[1] # Embed the labels into the message headers if self.synclabels: @@ -84,19 +83,23 @@ class GmailFolder(IMAPFolder): labels = labels - self.ignorelabels labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels)) - # First remove old label headers that may be in the message content retrieved + # First remove old label headers that may be in the message body retrieved # from gmail Then add a labels header with current gmail labels. - body = self.deletemessageheaders(body, self.labelsheader) - body = self.addmessageheader(body, '\n', self.labelsheader, labels_str) + self.deletemessageheaders(msg, self.labelsheader) + self.addmessageheader(msg, self.labelsheader, labels_str) - if len(body) > 200: - dbg_output = "%s...%s" % (str(body)[:150], str(body)[-50:]) - else: - dbg_output = body + if self.ui.is_debugging('imap'): + # Optimization: don't create the debugging objects unless needed + msg_s = msg.as_string(policy=self.policy['8bit-RFC']) + if len(msg_s) > 200: + dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:]) + else: + dbg_output = msg_s - self.ui.debug('imap', "Returned object from fetching %d: '%s'" % - (uid, dbg_output)) - return body + self.ui.debug('imap', "Returned object from fetching %d: '%s'" % + (uid, dbg_output)) + + return msg def getmessagelabels(self, uid): if 'labels' in self.messagelist[uid]: @@ -167,7 +170,7 @@ class GmailFolder(IMAPFolder): rtime = imaplibutil.Internaldate2epoch(messagestr) self.messagelist[uid] = {'uid': uid, 'flags': flags, 'labels': labels, 'time': rtime} - def savemessage(self, uid, content, flags, rtime): + def savemessage(self, uid, msg, flags, rtime): """Save the message on the Server This backend always assigns a new uid, so the uid arg is ignored. @@ -180,7 +183,7 @@ class GmailFolder(IMAPFolder): savemessage is never called in a dryrun mode. :param uid: Message UID - :param content: Message content + :param msg: Message object :param flags: Message flags :param rtime: A timestamp to be used as the mail date :returns: the UID of the new message as assigned by the server. If the @@ -189,13 +192,13 @@ class GmailFolder(IMAPFolder): read-only for example) it will return -1.""" if not self.synclabels: - return super(GmailFolder, self).savemessage(uid, content, flags, rtime) + return super(GmailFolder, self).savemessage(uid, msg, flags, rtime) labels = set() - for hstr in self.getmessageheaderlist(content, self.labelsheader): + for hstr in self.getmessageheaderlist(msg, self.labelsheader): labels.update(imaputil.labels_from_header(self.labelsheader, hstr)) - ret = super(GmailFolder, self).savemessage(uid, content, flags, rtime) + ret = super(GmailFolder, self).savemessage(uid, msg, flags, rtime) self.savemessagelabels(ret, labels) return ret diff --git a/offlineimap/folder/GmailMaildir.py b/offlineimap/folder/GmailMaildir.py index ebefd1b..1dfccc7 100644 --- a/offlineimap/folder/GmailMaildir.py +++ b/offlineimap/folder/GmailMaildir.py @@ -90,12 +90,12 @@ class GmailMaildirFolder(MaildirFolder): if not os.path.exists(filepath): return set() - file = open(filepath, 'rt') - content = file.read() - file.close() + fd = open(filepath, 'rb') + msg = self.parser['8bit'].parse(fd) + fd.close() self.messagelist[uid]['labels'] = set() - for hstr in self.getmessageheaderlist(content, self.labelsheader): + for hstr in self.getmessageheaderlist(msg, self.labelsheader): self.messagelist[uid]['labels'].update( imaputil.labels_from_header(self.labelsheader, hstr)) self.messagelist[uid]['labels_cached'] = True @@ -108,7 +108,7 @@ class GmailMaildirFolder(MaildirFolder): else: return self.messagelist[uid]['mtime'] - def savemessage(self, uid, content, flags, rtime): + def savemessage(self, uid, msg, flags, rtime): """Writes a new message, with the specified uid. See folder/Base for detail. Note that savemessage() does not @@ -116,14 +116,15 @@ class GmailMaildirFolder(MaildirFolder): savemessage is never called in a dryrun mode.""" if not self.synclabels: - return super(GmailMaildirFolder, self).savemessage(uid, content, + return super(GmailMaildirFolder, self).savemessage(uid, msg, flags, rtime) labels = set() - for hstr in self.getmessageheaderlist(content, self.labelsheader): + for hstr in self.getmessageheaderlist(msg, self.labelsheader): labels.update(imaputil.labels_from_header(self.labelsheader, hstr)) - ret = super(GmailMaildirFolder, self).savemessage(uid, content, flags, + # TODO - Not sure why the returned uid is stored early as ret here? + ret = super(GmailMaildirFolder, self).savemessage(uid, msg, flags, rtime) # Update the mtime and labels. @@ -145,12 +146,12 @@ class GmailMaildirFolder(MaildirFolder): filename = self.messagelist[uid]['filename'] filepath = os.path.join(self.getfullname(), filename) - file = open(filepath, 'rt') - content = file.read() - file.close() + fd = open(filepath, 'rb') + msg = self.parser['8bit'].parse(fd) + fd.close() oldlabels = set() - for hstr in self.getmessageheaderlist(content, self.labelsheader): + for hstr in self.getmessageheaderlist(msg, self.labelsheader): oldlabels.update(imaputil.labels_from_header(self.labelsheader, hstr)) @@ -167,15 +168,14 @@ class GmailMaildirFolder(MaildirFolder): sorted(labels | ignoredlabels)) # First remove old labels header, and then add the new one. - content = self.deletemessageheaders(content, self.labelsheader) - content = self.addmessageheader(content, '\n', self.labelsheader, - labels_str) + self.deletemessageheaders(msg, self.labelsheader) + self.addmessageheader(msg, self.labelsheader, labels_str) mtime = int(os.stat(filepath).st_mtime) # Write file with new labels to a unique file in tmp. messagename = self.new_message_filename(uid, set()) - tmpname = self.save_to_tmp_file(messagename, content) + tmpname = self.save_to_tmp_file(messagename, msg) tmppath = os.path.join(self.getfullname(), tmpname) # Move to actual location. diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index e81f73b..16d65da 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -342,19 +342,22 @@ class IMAPFolder(BaseFolder): data = self._fetch_from_imap(str(uid), self.retrycount) # Data looks now e.g. - # ['320 (17061 BODY[] {2565}','msgbody....'] + # ['320 (17061 BODY[] {2565}',] # Is a list of two elements. Message is at [1] - data = data[1].replace(CRLF, "\n") + msg = data[1] - if len(data) > 200: - dbg_output = "%s...%s" % (str(data)[:150], str(data)[-50:]) - else: - dbg_output = data + if self.ui.is_debugging('imap'): + # Optimization: don't create the debugging objects unless needed + msg_s = msg.as_string(policy=self.policy['8bit-RFC']) + if len(msg_s) > 200: + dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:]) + else: + dbg_output = msg_s - self.ui.debug('imap', "Returned object from fetching %d: '%s'" % - (uid, dbg_output)) + self.ui.debug('imap', "Returned object from fetching %d: '%s'" % + (uid, dbg_output)) - return data + return msg # Interface from BaseFolder def getmessagetime(self, uid): @@ -679,13 +682,15 @@ class IMAPFolder(BaseFolder): (headername, headervalue)) self.addmessageheader(msg, headername, headervalue) - msg_s = msg.as_string(policy=output_policy) - if len(msg_s) > 200: - dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:]) - else: - dbg_output = msg_s - self.ui.debug('imap', "savemessage: date: %s, content: '%s'" % - (date, dbg_output)) + if self.ui.is_debugging('imap'): + # Optimization: don't create the debugging objects unless needed + msg_s = msg.as_string(policy=output_policy) + if len(msg_s) > 200: + dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:]) + else: + dbg_output = msg_s + self.ui.debug('imap', "savemessage: date: %s, content: '%s'" % + (date, dbg_output)) try: # Select folder for append and make the box READ-WRITE. @@ -830,7 +835,7 @@ class IMAPFolder(BaseFolder): """Fetches data from IMAP server. Arguments: - - uids: message UIDS + - uids: message UIDS (OfflineIMAP3: First UID returned only) - retry_num: number of retries to make Returns: data obtained by this query.""" @@ -886,9 +891,21 @@ class IMAPFolder(BaseFolder): "with UID '%s'" % (self.getrepository(), uids) raise OfflineImapError(reason, severity) - # Convert bytes to str + # JI: In offlineimap, this function returned a tuple of strings for each + # fetched UID, offlineimap3 calls to the imap object return bytes and so + # originally a fixed, utf-8 conversion was done and *only* the first + # response (d[0]) was returned. Note that this alters the behavior + # between code bases. However, it seems like a single UID is the intent + # of this function so retaining the modfication here for now. + # + # TODO: Can we assume the server response containing the meta data is + # always 'utf-8' encoded? Assuming yes for now. + # + # Convert responses, d[0][0], into a 'utf-8' string (from bytes) and + # Convert email, d[0][1], into a message object (from bytes) + ndata0 = data[0][0].decode('utf-8') - ndata1 = data[0][1].decode('utf-8', errors='replace') + ndata1 = self.parser['8bit-RFC'].parsebytes(data[0][1]) ndata = [ndata0, ndata1] return ndata diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index abc0436..74a4ab2 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -259,7 +259,7 @@ class MaildirFolder(BaseFolder): filename = self.messagelist[uid]['filename'] filepath = os.path.join(self.getfullname(), filename) fd = open(filepath, 'rb') - retval = self.parse['8bit'](fd) + retval = self.parser['8bit'].parse(fd) fd.close() return retval diff --git a/offlineimap/ui/UIBase.py b/offlineimap/ui/UIBase.py index 1f0a8dd..d49d564 100644 --- a/offlineimap/ui/UIBase.py +++ b/offlineimap/ui/UIBase.py @@ -231,6 +231,9 @@ class UIBase: else: self.invaliddebug(debugtype) + def is_debugging(self, debugtype): + return (debugtype in self.debuglist) + def debugging(self, debugtype): global debugtypes self.logger.debug("Now debugging for %s: %s" % (debugtype,