Sync INTERNALDATE <-> mtime

The attached patch adds syncing the INTERNALDATE of IMAP folders with
the mtime of messages in maildir folders.
I want this to happen, because I'm running a dovecot over the maildirs
synced by offlineimap, and that uses the mtime as the INTERNALDATE.
When using mutt to view messages I generally sort based on the received
date, which for IMAP folders is the INTERNALDATE.

Since this is the first real coding I've done in Python the patch may
need to be cleaned up some, but it's working pretty well for me.  I've
added new messages to each side, and the received date has been
preserved going both ways.

offlineimap/folder/Base.py

@@ -133,7 +133,7 @@ class BaseFolder:
         """Returns the content of the specified message."""
         raise NotImplementedException
 
-    def savemessage(self, uid, content, flags):
+    def savemessage(self, uid, content, flags, rtime):
         """Writes a new message, with the specified uid.
         If the uid is < 0, the backend should assign a new uid and return it.
 
@@ -152,6 +152,10 @@ class BaseFolder:
         """
         raise NotImplementedException
 
+    def getmessagetime(self, uid):
+        """Return the received time for the specified message."""
+        raise NotImplementedException
+
     def getmessageflags(self, uid):
         """Returns the flags for the specified message."""
         raise NotImplementedException
@@ -203,8 +207,9 @@ class BaseFolder:
         successuid = None
         message = self.getmessage(uid)
         flags = self.getmessageflags(uid)
+        rtime = self.getmessagetime(uid)
         for tryappend in applyto:
-            successuid = tryappend.savemessage(uid, message, flags)
+            successuid = tryappend.savemessage(uid, message, flags, rtime)
             if successuid >= 0:
                 successobject = tryappend
                 break
@@ -214,10 +219,10 @@ class BaseFolder:
                 # Copy the message to the other remote servers.
                 for appendserver in \
                         [x for x in applyto if x != successobject]:
-                    appendserver.savemessage(successuid, message, flags)
+                    appendserver.savemessage(successuid, message, flags, rtime)
                     # Copy to its new name on the local server and delete
                     # the one without a UID.
-                    self.savemessage(successuid, message, flags)
+                    self.savemessage(successuid, message, flags, rtime)
             self.deletemessage(uid) # It'll be re-downloaded.
         else:
             # Did not find any server to take this message.  Ignore.
@@ -272,11 +277,12 @@ class BaseFolder:
                 message = self.getmessage(uid)
                 break
         flags = self.getmessageflags(uid)
+        rtime = self.getmessagetime(uid)
         for object in applyto:
-            newuid = object.savemessage(uid, message, flags)
+            newuid = object.savemessage(uid, message, flags, rtime)
             if newuid > 0 and newuid != uid:
                 # Change the local uid.
-                self.savemessage(newuid, message, flags)
+                self.savemessage(newuid, message, flags, rtime)
                 self.deletemessage(uid)
                 uid = newuid
         

offlineimap/folder/IMAP.py

@@ -84,7 +84,7 @@ class IMAPFolder(BaseFolder):
             # Now, get the flags and UIDs for these.
             # We could conceivably get rid of maxmsgid and just say
             # '1:*' here.
-            response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID)')[1]
+            response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID INTERNALDATE)')[1]
         finally:
             self.imapserver.releaseconnection(imapobj)
         for messagestr in response:
@@ -98,7 +98,8 @@ class IMAPFolder(BaseFolder):
             else:
                 uid = long(options['UID'])
                 flags = imaputil.flagsimap2maildir(options['FLAGS'])
-                self.messagelist[uid] = {'uid': uid, 'flags': flags}
+                rtime = imaplib.Internaldate2epoch(messagestr)
+                self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime}
 
     def getmessagelist(self):
         return self.messagelist
@@ -116,6 +117,9 @@ class IMAPFolder(BaseFolder):
         finally:
             self.imapserver.releaseconnection(imapobj)
 
+    def getmessagetime(self, uid):
+        return self.messagelist[uid]['time']
+    
     def getmessageflags(self, uid):
         return self.messagelist[uid]['flags']
 
@@ -177,7 +181,7 @@ class IMAPFolder(BaseFolder):
         matchinguids.sort()
         return long(matchinguids[0])
 
-    def savemessage(self, uid, content, flags):
+    def savemessage(self, uid, content, flags, rtime):
         imapobj = self.imapserver.acquireconnection()
         ui = UIBase.getglobalui()
         ui.debug('imap', 'savemessage: called')
@@ -193,11 +197,12 @@ class IMAPFolder(BaseFolder):
             # This backend always assigns a new uid, so the uid arg is ignored.
             # In order to get the new uid, we need to save off the message ID.
 
-            message = rfc822.Message(StringIO(content))
+            # If time isn't known
-            datetuple = rfc822.parsedate(message.getheader('Date'))
+            if rtime == None:
-            # Will be None if missing or not in a valid format.
-            if datetuple == None:
                 datetuple = time.localtime()
+            else:
+                datetuple = time.localtime(rtime)
+
             try:
                 if datetuple[0] < 1981:
                     raise ValueError

offlineimap/folder/LocalStatus.py

@@ -98,7 +98,7 @@ class LocalStatusFolder(BaseFolder):
     def getmessagelist(self):
         return self.messagelist
 
-    def savemessage(self, uid, content, flags):
+    def savemessage(self, uid, content, flags, rtime):
         if uid < 0:
             # We cannot assign a uid.
             return uid
@@ -107,13 +107,16 @@ class LocalStatusFolder(BaseFolder):
             self.savemessageflags(uid, flags)
             return uid
 
-        self.messagelist[uid] = {'uid': uid, 'flags': flags}
+        self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime}
         self.autosave()
         return uid
 
     def getmessageflags(self, uid):
         return self.messagelist[uid]['flags']
 
+    def getmessagetime(self, uid):
+        return self.messagelist[uid]['time']
+
     def savemessageflags(self, uid, flags):
         self.messagelist[uid]['flags'] = flags
         self.autosave()

offlineimap/folder/Maildir.py

@@ -124,7 +124,12 @@ class MaildirFolder(BaseFolder):
         file.close()
         return retval.replace("\r\n", "\n")
 
-    def savemessage(self, uid, content, flags):
+    def getmessagetime( self, uid ):
+        filename = self.messagelist[uid]['filename']
+        st = os.stat(filename)
+        return st.st_mtime
+
+    def savemessage(self, uid, content, flags, rtime):
         ui = UIBase.getglobalui()
         ui.debug('maildir', 'savemessage: called to write with flags %s and content %s' % \
                  (repr(flags), repr(content)))
@@ -165,6 +170,7 @@ class MaildirFolder(BaseFolder):
         file = open(os.path.join(tmpdir, tmpmessagename), "wt")
         file.write(content)
         file.close()
+        os.utime(os.path.join(tmpdir,tmpmessagename), (rtime,rtime))
         ui.debug('maildir', 'savemessage: moving from %s to %s' % \
                  (tmpmessagename, messagename))
         os.link(os.path.join(tmpdir, tmpmessagename),

offlineimap/folder/UIDMaps.py

@@ -130,7 +130,7 @@ class MappingFolderMixIn:
         """Returns the content of the specified message."""
         return self._mb.getmessage(self, self.r2l[uid])
 
-    def savemessage(self, uid, content, flags):
+    def savemessage(self, uid, content, flags, rtime):
         """Writes a new message, with the specified uid.
         If the uid is < 0, the backend should assign a new uid and return it.
 
@@ -153,7 +153,7 @@ class MappingFolderMixIn:
         if uid in self.r2l:
             self.savemessageflags(uid, flags)
             return uid
-        newluid = self._mb.savemessage(self, -1, content, flags)
+        newluid = self._mb.savemessage(self, -1, content, flags, rtime)
         if newluid < 1:
             raise ValueError, "Backend could not find uid for message"
         self.maplock.acquire()
@@ -169,6 +169,9 @@ class MappingFolderMixIn:
     def getmessageflags(self, uid):
         return self._mb.getmessageflags(self, self.r2l[uid])
 
+    def getmessagetime(self, uid):
+        return None
+
     def savemessageflags(self, uid, flags):
         self._mb.savemessageflags(self, self.r2l[uid], flags)
 

offlineimap/imaplib.py

@@ -5,6 +5,7 @@ Based on RFC 2060.
 Public class:           IMAP4
 Public variable:        Debug
 Public functions:       Internaldate2tuple
+                        Internaldate2epoch
                         Int2AP
                         ParseFlags
                         Time2Internaldate
@@ -24,7 +25,7 @@ __version__ = "2.52"
 import binascii, re, socket, time, random, sys, os
 from offlineimap.ui import UIBase
 
-__all__ = ["IMAP4", "Internaldate2tuple",
+__all__ = ["IMAP4", "Internaldate2tuple", "Internaldate2epoch",
            "Int2AP", "ParseFlags", "Time2Internaldate"]
 
 #       Globals
@@ -78,7 +79,7 @@ Commands = {
 Continuation = re.compile(r'\+( (?P<data>.*))?')
 Flags = re.compile(r'.*FLAGS \((?P<flags>[^\)]*)\)')
 InternalDate = re.compile(r'.*INTERNALDATE "'
-        r'(?P<day>[ 123][0-9])-(?P<mon>[A-Z][a-z][a-z])-(?P<year>[0-9][0-9][0-9][0-9])'
+        r'(?P<day>[ 0123][0-9])-(?P<mon>[A-Z][a-z][a-z])-(?P<year>[0-9][0-9][0-9][0-9])'
         r' (?P<hour>[0-9][0-9]):(?P<min>[0-9][0-9]):(?P<sec>[0-9][0-9])'
         r' (?P<zonen>[-+])(?P<zoneh>[0-9][0-9])(?P<zonem>[0-9][0-9])'
         r'"')
@@ -1230,10 +1231,10 @@ class _Authenticator:
 Mon2num = {'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'Jun': 6,
         'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12}
 
-def Internaldate2tuple(resp):
+def Internaldate2epoch(resp):
     """Convert IMAP4 INTERNALDATE to UT.
 
-    Returns Python time module tuple.
+    Returns seconds since the epoch.
     """
 
     mo = InternalDate.match(resp)
@@ -1259,7 +1260,16 @@ def Internaldate2tuple(resp):
 
     tt = (year, mon, day, hour, min, sec, -1, -1, -1)
 
-    utc = time.mktime(tt)
+    return time.mktime(tt)
+
+
+def Internaldate2tuple(resp):
+    """Convert IMAP4 INTERNALDATE to UT.
+
+    Returns Python time module tuple.
+    """
+
+    utc = Internaldate2epoch(resp)
 
     # Following is necessary because the time module has no 'mkgmtime'.
     # 'mktime' assumes arg in local timezone, so adds timezone/altzone.