add new config option filename_use_mail_timestamp

If this value is true, use (if possible) a timestamp based on message
Date or Delivery-date headers. The current system time is used
otherwise.

filename_use_mail_timestamp and utime_from_header are now completely
separated option that do not interfere one with other.

To handle this feature in a multithread context we use a hash to count
the number of mail with the same timestamp. This method is more accurate
than using the old lasttime and timeseq variables.

Signed-off-by: Sébastien Gross <seb•ɑƬ•chezwam•ɖɵʈ•org>
Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>

offlineimap/folder/Base.py

@@ -60,6 +60,13 @@ class BaseFolder(object):
         self._utime_from_header = self.config.getdefaultboolean(repo,
             "utime_from_header", utime_from_header_global)
 
+        # Do we need to use mail timestamp for filename prefix?
+        filename_use_mail_timestamp_global = self.config.getdefaultboolean(
+            "general", "filename_use_mail_timestamp", False)
+        repo = "Repository " + repository.name
+        self._filename_use_mail_timestamp = self.config.getdefaultboolean(repo,
+            "filename_use_mail_timestamp", filename_use_mail_timestamp_global)
+
         # Determine if we're running static or dynamic folder filtering
         # and check filtering status
         self._dynamic_folderfilter = self.config.getdefaultboolean(

offlineimap/folder/Maildir.py

@@ -38,22 +38,20 @@ re_uidmatch = re.compile(',U=(\d+)')
 # Find a numeric timestamp in a string (filename prefix)
 re_timestampmatch = re.compile('(\d+)');
 
-timeseq = 0
+timehash = {}
-lasttime = 0
 timelock = Lock()
 
-def _gettimeseq():
+def _gettimeseq(date=None):
-    global lasttime, timeseq, timelock
+    global timehash, timelock
     timelock.acquire()
     try:
-        thistime = long(time.time())
+        if date is None:
-        if thistime == lasttime:
+            date = long(time.time())
-            timeseq += 1
+        if timehash.has_key(date):
-            return (thistime, timeseq)
+            timehash[date] += 1
         else:
-            lasttime = thistime
+            timehash[date] = 0
-            timeseq = 0
+        return (date, timehash[date])
-            return (thistime, timeseq)
     finally:
         timelock.release()
 
@@ -269,14 +267,14 @@ class MaildirFolder(BaseFolder):
         filepath = os.path.join(self.getfullname(), filename)
         return os.path.getmtime(filepath)
 
-    def new_message_filename(self, uid, flags=set()):
+    def new_message_filename(self, uid, flags=set(), date=None):
         """Creates a new unique Maildir filename
 
         :param uid: The UID`None`, or a set of maildir flags
         :param flags: A set of maildir flags
         :returns: String containing unique message filename"""
 
-        timeval, timeseq = _gettimeseq()
+        timeval, timeseq = _gettimeseq(date)
         return '%d_%d.%d.%s,U=%d,FMD5=%s%s2,%s'% \
             (timeval, timeseq, os.getpid(), socket.gethostname(),
             uid, self._foldermd5, self.infosep, ''.join(sorted(flags)))
@@ -346,7 +344,27 @@ class MaildirFolder(BaseFolder):
         # Otherwise, save the message in tmp/ and then call savemessageflags()
         # to give it a permanent home.
         tmpdir = os.path.join(self.getfullname(), 'tmp')
-        messagename = self.new_message_filename(uid, flags)
+
+        # use the mail timestamp given by either Date or Delivery-date mail
+        # headers.
+        message_timestamp = None
+        if self._filename_use_mail_timestamp:
+            try:
+                message_timestamp = emailutil.get_message_date(content, 'Date')
+                if message_timestamp is None:
+                    # Give a try with Delivery-date
+                    date = emailutil.get_message_date(content, 'Delivery-date')
+            except:
+                # This should never happen
+                from email.Parser import Parser
+                from offlineimap.ui import getglobalui
+                datestr = Parser().parsestr(content, True).get("Date")
+                ui = getglobalui()
+                ui.warn("UID %d has invalid date %s: %s\n"
+                    "Not using message timestamp as file prefix" % (uid, datestr, e))
+                # No need to check if date is None here since it would
+                # be overridden by _gettimeseq.
+        messagename = self.new_message_filename(uid, flags, date=message_timestamp)
         tmpname = self.save_to_tmp_file(messagename, content)
 
         if self.utime_from_header: