From 03488ba81b0afc239ee88701384c765cd68db8db Mon Sep 17 00:00:00 2001 From: John Goerzen Date: Tue, 22 Aug 2006 02:09:36 +0100 Subject: [PATCH] Sync INTERNALDATE <-> mtime The attached patch adds syncing the INTERNALDATE of IMAP folders with the mtime of messages in maildir folders. I want this to happen, because I'm running a dovecot over the maildirs synced by offlineimap, and that uses the mtime as the INTERNALDATE. When using mutt to view messages I generally sort based on the received date, which for IMAP folders is the INTERNALDATE. Since this is the first real coding I've done in Python the patch may need to be cleaned up some, but it's working pretty well for me. I've added new messages to each side, and the received date has been preserved going both ways. --- offlineimap/folder/Base.py | 18 ++++++++++++------ offlineimap/folder/IMAP.py | 19 ++++++++++++------- offlineimap/folder/LocalStatus.py | 7 +++++-- offlineimap/folder/Maildir.py | 8 +++++++- offlineimap/folder/UIDMaps.py | 7 +++++-- offlineimap/imaplib.py | 20 +++++++++++++++----- 6 files changed, 56 insertions(+), 23 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 0ee1412..9b29ab3 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -133,7 +133,7 @@ class BaseFolder: """Returns the content of the specified message.""" raise NotImplementedException - def savemessage(self, uid, content, flags): + def savemessage(self, uid, content, flags, rtime): """Writes a new message, with the specified uid. If the uid is < 0, the backend should assign a new uid and return it. @@ -152,6 +152,10 @@ class BaseFolder: """ raise NotImplementedException + def getmessagetime(self, uid): + """Return the received time for the specified message.""" + raise NotImplementedException + def getmessageflags(self, uid): """Returns the flags for the specified message.""" raise NotImplementedException @@ -203,8 +207,9 @@ class BaseFolder: successuid = None message = self.getmessage(uid) flags = self.getmessageflags(uid) + rtime = self.getmessagetime(uid) for tryappend in applyto: - successuid = tryappend.savemessage(uid, message, flags) + successuid = tryappend.savemessage(uid, message, flags, rtime) if successuid >= 0: successobject = tryappend break @@ -214,10 +219,10 @@ class BaseFolder: # Copy the message to the other remote servers. for appendserver in \ [x for x in applyto if x != successobject]: - appendserver.savemessage(successuid, message, flags) + appendserver.savemessage(successuid, message, flags, rtime) # Copy to its new name on the local server and delete # the one without a UID. - self.savemessage(successuid, message, flags) + self.savemessage(successuid, message, flags, rtime) self.deletemessage(uid) # It'll be re-downloaded. else: # Did not find any server to take this message. Ignore. @@ -272,11 +277,12 @@ class BaseFolder: message = self.getmessage(uid) break flags = self.getmessageflags(uid) + rtime = self.getmessagetime(uid) for object in applyto: - newuid = object.savemessage(uid, message, flags) + newuid = object.savemessage(uid, message, flags, rtime) if newuid > 0 and newuid != uid: # Change the local uid. - self.savemessage(newuid, message, flags) + self.savemessage(newuid, message, flags, rtime) self.deletemessage(uid) uid = newuid diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 7cb334a..8a4f7e4 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -84,7 +84,7 @@ class IMAPFolder(BaseFolder): # Now, get the flags and UIDs for these. # We could conceivably get rid of maxmsgid and just say # '1:*' here. - response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID)')[1] + response = imapobj.fetch('1:%d' % maxmsgid, '(FLAGS UID INTERNALDATE)')[1] finally: self.imapserver.releaseconnection(imapobj) for messagestr in response: @@ -98,7 +98,8 @@ class IMAPFolder(BaseFolder): else: uid = long(options['UID']) flags = imaputil.flagsimap2maildir(options['FLAGS']) - self.messagelist[uid] = {'uid': uid, 'flags': flags} + rtime = imaplib.Internaldate2epoch(messagestr) + self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} def getmessagelist(self): return self.messagelist @@ -115,6 +116,9 @@ class IMAPFolder(BaseFolder): finally: self.imapserver.releaseconnection(imapobj) + + def getmessagetime(self, uid): + return self.messagelist[uid]['time'] def getmessageflags(self, uid): return self.messagelist[uid]['flags'] @@ -177,7 +181,7 @@ class IMAPFolder(BaseFolder): matchinguids.sort() return long(matchinguids[0]) - def savemessage(self, uid, content, flags): + def savemessage(self, uid, content, flags, rtime): imapobj = self.imapserver.acquireconnection() ui = UIBase.getglobalui() ui.debug('imap', 'savemessage: called') @@ -193,11 +197,12 @@ class IMAPFolder(BaseFolder): # This backend always assigns a new uid, so the uid arg is ignored. # In order to get the new uid, we need to save off the message ID. - message = rfc822.Message(StringIO(content)) - datetuple = rfc822.parsedate(message.getheader('Date')) - # Will be None if missing or not in a valid format. - if datetuple == None: + # If time isn't known + if rtime == None: datetuple = time.localtime() + else: + datetuple = time.localtime(rtime) + try: if datetuple[0] < 1981: raise ValueError diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index 1de7bdd..937825d 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -98,7 +98,7 @@ class LocalStatusFolder(BaseFolder): def getmessagelist(self): return self.messagelist - def savemessage(self, uid, content, flags): + def savemessage(self, uid, content, flags, rtime): if uid < 0: # We cannot assign a uid. return uid @@ -107,13 +107,16 @@ class LocalStatusFolder(BaseFolder): self.savemessageflags(uid, flags) return uid - self.messagelist[uid] = {'uid': uid, 'flags': flags} + self.messagelist[uid] = {'uid': uid, 'flags': flags, 'time': rtime} self.autosave() return uid def getmessageflags(self, uid): return self.messagelist[uid]['flags'] + def getmessagetime(self, uid): + return self.messagelist[uid]['time'] + def savemessageflags(self, uid, flags): self.messagelist[uid]['flags'] = flags self.autosave() diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 525f936..6d61c3d 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -124,7 +124,12 @@ class MaildirFolder(BaseFolder): file.close() return retval.replace("\r\n", "\n") - def savemessage(self, uid, content, flags): + def getmessagetime( self, uid ): + filename = self.messagelist[uid]['filename'] + st = os.stat(filename) + return st.st_mtime + + def savemessage(self, uid, content, flags, rtime): ui = UIBase.getglobalui() ui.debug('maildir', 'savemessage: called to write with flags %s and content %s' % \ (repr(flags), repr(content))) @@ -165,6 +170,7 @@ class MaildirFolder(BaseFolder): file = open(os.path.join(tmpdir, tmpmessagename), "wt") file.write(content) file.close() + os.utime(os.path.join(tmpdir,tmpmessagename), (rtime,rtime)) ui.debug('maildir', 'savemessage: moving from %s to %s' % \ (tmpmessagename, messagename)) os.link(os.path.join(tmpdir, tmpmessagename), diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 9cd35d3..946f1f5 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -130,7 +130,7 @@ class MappingFolderMixIn: """Returns the content of the specified message.""" return self._mb.getmessage(self, self.r2l[uid]) - def savemessage(self, uid, content, flags): + def savemessage(self, uid, content, flags, rtime): """Writes a new message, with the specified uid. If the uid is < 0, the backend should assign a new uid and return it. @@ -153,7 +153,7 @@ class MappingFolderMixIn: if uid in self.r2l: self.savemessageflags(uid, flags) return uid - newluid = self._mb.savemessage(self, -1, content, flags) + newluid = self._mb.savemessage(self, -1, content, flags, rtime) if newluid < 1: raise ValueError, "Backend could not find uid for message" self.maplock.acquire() @@ -169,6 +169,9 @@ class MappingFolderMixIn: def getmessageflags(self, uid): return self._mb.getmessageflags(self, self.r2l[uid]) + def getmessagetime(self, uid): + return None + def savemessageflags(self, uid, flags): self._mb.savemessageflags(self, self.r2l[uid], flags) diff --git a/offlineimap/imaplib.py b/offlineimap/imaplib.py index 4d44247..2bbe92e 100644 --- a/offlineimap/imaplib.py +++ b/offlineimap/imaplib.py @@ -5,6 +5,7 @@ Based on RFC 2060. Public class: IMAP4 Public variable: Debug Public functions: Internaldate2tuple + Internaldate2epoch Int2AP ParseFlags Time2Internaldate @@ -24,7 +25,7 @@ __version__ = "2.52" import binascii, re, socket, time, random, sys, os from offlineimap.ui import UIBase -__all__ = ["IMAP4", "Internaldate2tuple", +__all__ = ["IMAP4", "Internaldate2tuple", "Internaldate2epoch", "Int2AP", "ParseFlags", "Time2Internaldate"] # Globals @@ -78,7 +79,7 @@ Commands = { Continuation = re.compile(r'\+( (?P.*))?') Flags = re.compile(r'.*FLAGS \((?P[^\)]*)\)') InternalDate = re.compile(r'.*INTERNALDATE "' - r'(?P[ 123][0-9])-(?P[A-Z][a-z][a-z])-(?P[0-9][0-9][0-9][0-9])' + r'(?P[ 0123][0-9])-(?P[A-Z][a-z][a-z])-(?P[0-9][0-9][0-9][0-9])' r' (?P[0-9][0-9]):(?P[0-9][0-9]):(?P[0-9][0-9])' r' (?P[-+])(?P[0-9][0-9])(?P[0-9][0-9])' r'"') @@ -1230,10 +1231,10 @@ class _Authenticator: Mon2num = {'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'Jun': 6, 'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12} -def Internaldate2tuple(resp): +def Internaldate2epoch(resp): """Convert IMAP4 INTERNALDATE to UT. - Returns Python time module tuple. + Returns seconds since the epoch. """ mo = InternalDate.match(resp) @@ -1259,7 +1260,16 @@ def Internaldate2tuple(resp): tt = (year, mon, day, hour, min, sec, -1, -1, -1) - utc = time.mktime(tt) + return time.mktime(tt) + + +def Internaldate2tuple(resp): + """Convert IMAP4 INTERNALDATE to UT. + + Returns Python time module tuple. + """ + + utc = Internaldate2epoch(resp) # Following is necessary because the time module has no 'mkgmtime'. # 'mktime' assumes arg in local timezone, so adds timezone/altzone.