Fixing up the rest of the parsing and IMAP functions, and GMAIL classes as well. Also adding is_debugging() to the UI to allow us to quickly determine if we should build some of the expensive debug objects

offlineimap/folder/Base.py

@@ -60,9 +60,9 @@ class BaseFolder:
           policy.default.clone(cte_type='8bit', utf8=True, refold_source='none', linesep='\r\n'),
         }
         # Parsers
-        self.parse = {
+        self.parser = {}
-          '8bit': BytesParser(policy=p1),
+        for key in self.policy:
-        }
+            self.parser[key] = BytesParser(policy=self.policy[key])
         # Save original name for folderfilter operations.
         self.ffilter_name = name
         # Top level dir name is always ''.

offlineimap/folder/Gmail.py

@@ -69,10 +69,9 @@ class GmailFolder(IMAPFolder):
         data = self._fetch_from_imap(str(uid), self.retrycount)
 
         # data looks now e.g.
-        # ['320 (X-GM-LABELS (...) UID 17061 BODY[] {2565}','msgbody....']
+        # ['320 (X-GM-LABELS (...) UID 17061 BODY[] {2565}',<email.message.EmailMessage object>]
         # we only asked for one message, and that msg is in data[1].
-        # msbody is in [1].
+        msg = data[1]
-        body = data[1].replace("\r\n", "\n")
 
         # Embed the labels into the message headers
         if self.synclabels:
@@ -84,19 +83,23 @@ class GmailFolder(IMAPFolder):
             labels = labels - self.ignorelabels
             labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels))
 
-            # First remove old label headers that may be in the message content retrieved
+            # First remove old label headers that may be in the message body retrieved
             # from gmail Then add a labels header with current gmail labels.
-            body = self.deletemessageheaders(body, self.labelsheader)
+            self.deletemessageheaders(msg, self.labelsheader)
-            body = self.addmessageheader(body, '\n', self.labelsheader, labels_str)
+            self.addmessageheader(msg, self.labelsheader, labels_str)
 
-        if len(body) > 200:
+        if self.ui.is_debugging('imap'):
-            dbg_output = "%s...%s" % (str(body)[:150], str(body)[-50:])
+            # Optimization: don't create the debugging objects unless needed
-        else:
+            msg_s = msg.as_string(policy=self.policy['8bit-RFC'])
-            dbg_output = body
+            if len(msg_s) > 200:
+                dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:])
+            else:
+                dbg_output = msg_s
 
-        self.ui.debug('imap', "Returned object from fetching %d: '%s'" %
+            self.ui.debug('imap', "Returned object from fetching %d: '%s'" %
-                      (uid, dbg_output))
+                          (uid, dbg_output))
-        return body
+
+        return msg
 
     def getmessagelabels(self, uid):
         if 'labels' in self.messagelist[uid]:
@@ -167,7 +170,7 @@ class GmailFolder(IMAPFolder):
                 rtime = imaplibutil.Internaldate2epoch(messagestr)
                 self.messagelist[uid] = {'uid': uid, 'flags': flags, 'labels': labels, 'time': rtime}
 
-    def savemessage(self, uid, content, flags, rtime):
+    def savemessage(self, uid, msg, flags, rtime):
         """Save the message on the Server
 
         This backend always assigns a new uid, so the uid arg is ignored.
@@ -180,7 +183,7 @@ class GmailFolder(IMAPFolder):
         savemessage is never called in a dryrun mode.
 
         :param uid: Message UID
-        :param content: Message content
+        :param msg: Message object
         :param flags: Message flags
         :param rtime: A timestamp to be used as the mail date
         :returns: the UID of the new message as assigned by the server. If the
@@ -189,13 +192,13 @@ class GmailFolder(IMAPFolder):
                   read-only for example) it will return -1."""
 
         if not self.synclabels:
-            return super(GmailFolder, self).savemessage(uid, content, flags, rtime)
+            return super(GmailFolder, self).savemessage(uid, msg, flags, rtime)
 
         labels = set()
-        for hstr in self.getmessageheaderlist(content, self.labelsheader):
+        for hstr in self.getmessageheaderlist(msg, self.labelsheader):
             labels.update(imaputil.labels_from_header(self.labelsheader, hstr))
 
-        ret = super(GmailFolder, self).savemessage(uid, content, flags, rtime)
+        ret = super(GmailFolder, self).savemessage(uid, msg, flags, rtime)
         self.savemessagelabels(ret, labels)
         return ret
 

offlineimap/folder/GmailMaildir.py

@@ -90,12 +90,12 @@ class GmailMaildirFolder(MaildirFolder):
             if not os.path.exists(filepath):
                 return set()
 
-            file = open(filepath, 'rt')
+            fd = open(filepath, 'rb')
-            content = file.read()
+            msg = self.parser['8bit'].parse(fd)
-            file.close()
+            fd.close()
 
             self.messagelist[uid]['labels'] = set()
-            for hstr in self.getmessageheaderlist(content, self.labelsheader):
+            for hstr in self.getmessageheaderlist(msg, self.labelsheader):
                 self.messagelist[uid]['labels'].update(
                     imaputil.labels_from_header(self.labelsheader, hstr))
             self.messagelist[uid]['labels_cached'] = True
@@ -108,7 +108,7 @@ class GmailMaildirFolder(MaildirFolder):
         else:
             return self.messagelist[uid]['mtime']
 
-    def savemessage(self, uid, content, flags, rtime):
+    def savemessage(self, uid, msg, flags, rtime):
         """Writes a new message, with the specified uid.
 
         See folder/Base for detail. Note that savemessage() does not
@@ -116,14 +116,15 @@ class GmailMaildirFolder(MaildirFolder):
         savemessage is never called in a dryrun mode."""
 
         if not self.synclabels:
-            return super(GmailMaildirFolder, self).savemessage(uid, content,
+            return super(GmailMaildirFolder, self).savemessage(uid, msg,
                                                                flags, rtime)
 
         labels = set()
-        for hstr in self.getmessageheaderlist(content, self.labelsheader):
+        for hstr in self.getmessageheaderlist(msg, self.labelsheader):
             labels.update(imaputil.labels_from_header(self.labelsheader, hstr))
 
-        ret = super(GmailMaildirFolder, self).savemessage(uid, content, flags,
+        # TODO - Not sure why the returned uid is stored early as ret here?
+        ret = super(GmailMaildirFolder, self).savemessage(uid, msg, flags,
                                                           rtime)
 
         # Update the mtime and labels.
@@ -145,12 +146,12 @@ class GmailMaildirFolder(MaildirFolder):
         filename = self.messagelist[uid]['filename']
         filepath = os.path.join(self.getfullname(), filename)
 
-        file = open(filepath, 'rt')
+        fd = open(filepath, 'rb')
-        content = file.read()
+        msg = self.parser['8bit'].parse(fd)
-        file.close()
+        fd.close()
 
         oldlabels = set()
-        for hstr in self.getmessageheaderlist(content, self.labelsheader):
+        for hstr in self.getmessageheaderlist(msg, self.labelsheader):
             oldlabels.update(imaputil.labels_from_header(self.labelsheader,
                                                          hstr))
 
@@ -167,15 +168,14 @@ class GmailMaildirFolder(MaildirFolder):
                                                    sorted(labels | ignoredlabels))
 
         # First remove old labels header, and then add the new one.
-        content = self.deletemessageheaders(content, self.labelsheader)
+        self.deletemessageheaders(msg, self.labelsheader)
-        content = self.addmessageheader(content, '\n', self.labelsheader,
+        self.addmessageheader(msg, self.labelsheader, labels_str)
-                                        labels_str)
 
         mtime = int(os.stat(filepath).st_mtime)
 
         # Write file with new labels to a unique file in tmp.
         messagename = self.new_message_filename(uid, set())
-        tmpname = self.save_to_tmp_file(messagename, content)
+        tmpname = self.save_to_tmp_file(messagename, msg)
         tmppath = os.path.join(self.getfullname(), tmpname)
 
         # Move to actual location.

offlineimap/folder/IMAP.py

@@ -342,19 +342,22 @@ class IMAPFolder(BaseFolder):
         data = self._fetch_from_imap(str(uid), self.retrycount)
 
         # Data looks now e.g.
-        # ['320 (17061 BODY[] {2565}','msgbody....']
+        # ['320 (17061 BODY[] {2565}',<email.message.EmailMessage object>]
         # Is a list of two elements. Message is at [1]
-        data = data[1].replace(CRLF, "\n")
+        msg = data[1]
 
-        if len(data) > 200:
+        if self.ui.is_debugging('imap'):
-            dbg_output = "%s...%s" % (str(data)[:150], str(data)[-50:])
+            # Optimization: don't create the debugging objects unless needed
-        else:
+            msg_s = msg.as_string(policy=self.policy['8bit-RFC'])
-            dbg_output = data
+            if len(msg_s) > 200:
+                dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:])
+            else:
+                dbg_output = msg_s
 
-        self.ui.debug('imap', "Returned object from fetching %d: '%s'" %
+            self.ui.debug('imap', "Returned object from fetching %d: '%s'" %
-                      (uid, dbg_output))
+                          (uid, dbg_output))
 
-        return data
+        return msg
 
     # Interface from BaseFolder
     def getmessagetime(self, uid):
@@ -679,13 +682,15 @@ class IMAPFolder(BaseFolder):
                                   (headername, headervalue))
                     self.addmessageheader(msg, headername, headervalue)
 
-                msg_s = msg.as_string(policy=output_policy)
+                if self.ui.is_debugging('imap'):
-                if len(msg_s) > 200:
+                    # Optimization: don't create the debugging objects unless needed
-                    dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:])
+                    msg_s = msg.as_string(policy=output_policy)
-                else:
+                    if len(msg_s) > 200:
-                    dbg_output = msg_s
+                        dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:])
-                self.ui.debug('imap', "savemessage: date: %s, content: '%s'" %
+                    else:
-                              (date, dbg_output))
+                        dbg_output = msg_s
+                    self.ui.debug('imap', "savemessage: date: %s, content: '%s'" %
+                                  (date, dbg_output))
 
                 try:
                     # Select folder for append and make the box READ-WRITE.
@@ -830,7 +835,7 @@ class IMAPFolder(BaseFolder):
         """Fetches data from IMAP server.
 
         Arguments:
-        - uids: message UIDS
+        - uids: message UIDS (OfflineIMAP3: First UID returned only)
         - retry_num: number of retries to make
 
         Returns: data obtained by this query."""
@@ -886,9 +891,21 @@ class IMAPFolder(BaseFolder):
                          "with UID '%s'" % (self.getrepository(), uids)
             raise OfflineImapError(reason, severity)
 
-        # Convert bytes to str
+        # JI: In offlineimap, this function returned a tuple of strings for each
+        # fetched UID, offlineimap3 calls to the imap object return bytes and so
+        # originally a fixed, utf-8 conversion was done and *only* the first
+        # response (d[0]) was returned.  Note that this alters the behavior
+        # between code bases.  However, it seems like a single UID is the intent
+        # of this function so retaining the modfication here for now.
+        # 
+        # TODO: Can we assume the server response containing the meta data is
+        # always 'utf-8' encoded?  Assuming yes for now.
+        #
+        # Convert responses, d[0][0], into a 'utf-8' string (from bytes) and
+        # Convert email, d[0][1], into a message object (from bytes) 
+
         ndata0 = data[0][0].decode('utf-8')
-        ndata1 = data[0][1].decode('utf-8', errors='replace')
+        ndata1 = self.parser['8bit-RFC'].parsebytes(data[0][1])
         ndata = [ndata0, ndata1]
 
         return ndata

offlineimap/folder/Maildir.py

@@ -259,7 +259,7 @@ class MaildirFolder(BaseFolder):
         filename = self.messagelist[uid]['filename']
         filepath = os.path.join(self.getfullname(), filename)
         fd = open(filepath, 'rb')
-        retval = self.parse['8bit'](fd)
+        retval = self.parser['8bit'].parse(fd)
         fd.close()
         return retval
 

offlineimap/ui/UIBase.py

@@ -231,6 +231,9 @@ class UIBase:
         else:
             self.invaliddebug(debugtype)
 
+    def is_debugging(self, debugtype):
+        return (debugtype in self.debuglist)
+
     def debugging(self, debugtype):
         global debugtypes
         self.logger.debug("Now debugging for %s: %s" % (debugtype,