Merge branch 'renard-timestap-maildir' into next

offlineimap.conf

@@ -514,6 +514,29 @@ localfolders = ~/Test
 #utime_from_header = no
 
 
+# This option stands in the [Repository LocalExample] section.
+#
+# This option is similar to "utime_from_header" and could be use as a
+# complementary feature to keep track of a message date. This option only
+# makes sense for the Maildir type.
+#
+# By default each message is stored in a file which prefix is the fetch
+# timestamp and an order rank such as "1446590057_0". In a multithreading
+# environment message are fetched in a random order, then you can't trust
+# the file name to sort your boxes.
+#
+# If set to "yes" the file name prefix if build on the message "Date" header
+# (which should be present) or the "Received-date" if "Date" is not
+# found. If neither "Received-date" nor "Date" is found, the current system
+# date is used.  Now you can quickly sort your messages using their file
+# names.
+#
+# Used in combination with "utime_from_header" all your message would be in
+# order with the correct mtime attribute.
+#
+#filename_use_mail_timestamp = no
+
+
 [Repository GmailLocalExample]
 
 # This type of repository enables syncing of Gmail. All Maildir

offlineimap/folder/Base.py

@@ -60,6 +60,13 @@ class BaseFolder(object):
         self._utime_from_header = self.config.getdefaultboolean(repo,
             "utime_from_header", utime_from_header_global)
 
+        # Do we need to use mail timestamp for filename prefix?
+        filename_use_mail_timestamp_global = self.config.getdefaultboolean(
+            "general", "filename_use_mail_timestamp", False)
+        repo = "Repository " + repository.name
+        self._filename_use_mail_timestamp = self.config.getdefaultboolean(repo,
+            "filename_use_mail_timestamp", filename_use_mail_timestamp_global)
+
         # Determine if we're running static or dynamic folder filtering
         # and check filtering status
         self._dynamic_folderfilter = self.config.getdefaultboolean(

offlineimap/folder/Maildir.py

@@ -38,22 +38,20 @@ re_uidmatch = re.compile(',U=(\d+)')
 # Find a numeric timestamp in a string (filename prefix)
 re_timestampmatch = re.compile('(\d+)');
 
-timeseq = 0
+timehash = {}
-lasttime = 0
 timelock = Lock()
 
-def _gettimeseq():
+def _gettimeseq(date=None):
-    global lasttime, timeseq, timelock
+    global timehash, timelock
     timelock.acquire()
     try:
-        thistime = long(time.time())
+        if date is None:
-        if thistime == lasttime:
+            date = long(time.time())
-            timeseq += 1
+        if timehash.has_key(date):
-            return (thistime, timeseq)
+            timehash[date] += 1
         else:
-            lasttime = thistime
+            timehash[date] = 0
-            timeseq = 0
+        return (date, timehash[date])
-            return (thistime, timeseq)
     finally:
         timelock.release()
 
@@ -269,14 +267,14 @@ class MaildirFolder(BaseFolder):
         filepath = os.path.join(self.getfullname(), filename)
         return os.path.getmtime(filepath)
 
-    def new_message_filename(self, uid, flags=set()):
+    def new_message_filename(self, uid, flags=set(), date=None):
         """Creates a new unique Maildir filename
 
         :param uid: The UID`None`, or a set of maildir flags
         :param flags: A set of maildir flags
         :returns: String containing unique message filename"""
 
-        timeval, timeseq = _gettimeseq()
+        timeval, timeseq = _gettimeseq(date)
         return '%d_%d.%d.%s,U=%d,FMD5=%s%s2,%s'% \
             (timeval, timeseq, os.getpid(), socket.gethostname(),
             uid, self._foldermd5, self.infosep, ''.join(sorted(flags)))
@@ -346,7 +344,27 @@ class MaildirFolder(BaseFolder):
         # Otherwise, save the message in tmp/ and then call savemessageflags()
         # to give it a permanent home.
         tmpdir = os.path.join(self.getfullname(), 'tmp')
-        messagename = self.new_message_filename(uid, flags)
+
+        # use the mail timestamp given by either Date or Delivery-date mail
+        # headers.
+        message_timestamp = None
+        if self._filename_use_mail_timestamp:
+            try:
+                message_timestamp = emailutil.get_message_date(content, 'Date')
+                if message_timestamp is None:
+                    # Give a try with Delivery-date
+                    date = emailutil.get_message_date(content, 'Delivery-date')
+            except:
+                # This should never happen
+                from email.Parser import Parser
+                from offlineimap.ui import getglobalui
+                datestr = Parser().parsestr(content, True).get("Date")
+                ui = getglobalui()
+                ui.warn("UID %d has invalid date %s: %s\n"
+                    "Not using message timestamp as file prefix" % (uid, datestr, e))
+                # No need to check if date is None here since it would
+                # be overridden by _gettimeseq.
+        messagename = self.new_message_filename(uid, flags, date=message_timestamp)
         tmpname = self.save_to_tmp_file(messagename, content)
 
         if self.utime_from_header: