From d8026c5308d25aeafd8ef6c6804e1ea757da1ff2 Mon Sep 17 00:00:00 2001
From: Sebastian Spaeth <Sebastian@SSpaeth.de>
Date: Mon, 13 Jun 2011 15:22:37 +0200
Subject: [PATCH] Simplify Maildir message saving

Previously we were attempting to save out mails according to
 http://www.qmail.org/man/man5/maildir.html in 4 steps:

1 Create a unique filename
2 Do stat(tmp/<filename>). If it found a file, wait 2 sec and go back to 1.
3 Create and write the message to the tmp/<filename>.
4 Link from tmp/* to new/*

(we did step 2 up to 15 times) But as stated by
http://wiki1.dovecot.org/MailboxFormat/Maildir (see section 'Issues with
the specification'), this is a pointless approach, e.g. there are race
issues between stating that the filename does not exist and the actual
moving (when it might exist).

So, we can simplify the steps as suggested in the dovecot wiki and
tighten up our safety at the same time.

One improvement that we do is to open the file, guaranteeing that it did
not exist before in an atomic manner, thus our simplified approach is
really more secure than what we had before.

Also, we throw an OfflineImapError at MESSAGE level when the supposedly
unique filename already exists, so that we can skip this message and
still continue with other messages.

Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de>
Signed-off-by: Nicolas Sebrecht <nicolas.s-dev@laposte.net>
---
 Changelog.draft.rst           |  3 ++
 offlineimap/folder/Maildir.py | 63 ++++++++++++++---------------------
 2 files changed, 28 insertions(+), 38 deletions(-)

diff --git a/Changelog.draft.rst b/Changelog.draft.rst
index fd445ac..2035aa8 100644
--- a/Changelog.draft.rst
+++ b/Changelog.draft.rst
@@ -16,9 +16,12 @@ New Features
 Changes
 -------
 
+* Maildirs use less memory while syncing.
+
 Bug Fixes
 ---------
 
+* Saving to Maildirs now checks for file existence without race conditions.
 
 Pending for the next major release
 ==================================
diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py
index a798b36..b576507 100644
--- a/offlineimap/folder/Maildir.py
+++ b/offlineimap/folder/Maildir.py
@@ -28,6 +28,8 @@ try:
 except ImportError:
     from md5 import md5
 
+from offlineimap import OfflineImapError
+
 uidmatchre = re.compile(',U=(\d+)')
 flagmatchre = re.compile(':.*2,([A-Z]+)')
 timestampmatchre = re.compile('(\d+)');
@@ -217,52 +219,37 @@ class MaildirFolder(BaseFolder):
         # Otherwise, save the message in tmp/ and then call savemessageflags()
         # to give it a permanent home.
         tmpdir = os.path.join(self.getfullname(), 'tmp')
-        messagename = None
-        attempts = 0
-        while 1:
-            if attempts > 15:
-                raise IOError, "Couldn't write to file %s" % messagename
-            timeval, timeseq = gettimeseq()
-            messagename = '%d_%d.%d.%s,U=%d,FMD5=%s' % \
-                          (timeval,
-                           timeseq,
-                           os.getpid(),
-                           socket.gethostname(),
-                           uid,
-                           md5(self.getvisiblename()).hexdigest())
-            if os.path.exists(os.path.join(tmpdir, messagename)):
-                time.sleep(2)
-                attempts += 1
+        timeval, timeseq = gettimeseq()
+        messagename = '%d_%d.%d.%s,U=%d,FMD5=%s' % \
+            (timeval,
+             timeseq,
+             os.getpid(),
+             socket.gethostname(),
+             uid,
+             md5(self.getvisiblename()).hexdigest())
+        # open file and write it out
+        try:
+            fd = os.open(os.path.join(tmpdir, messagename),
+                           os.O_EXCL|os.O_CREAT|os.O_WRONLY)
+        except OSError, e:
+            if e.errno == 17: 
+                #FILE EXISTS ALREADY
+                severity = OfflineImapError.ERROR.MESSAGE
+                raise OfflineImapError("Unique filename %s already existing." %\
+                                           messagename, severity)
             else:
-                break
-        tmpmessagename = messagename.split(',')[0]
-        self.ui.debug('maildir', 'savemessage: using temporary name %s' %\
-                          tmpmessagename)
-        file = open(os.path.join(tmpdir, tmpmessagename), "wt")
-        file.write(content)
+                raise
 
+        file = os.fdopen(fd, 'wt')
+        file.write(content)
         # Make sure the data hits the disk
         file.flush()
         if self.dofsync:
-            os.fsync(file.fileno())
+            os.fsync(fd)
         file.close()
 
         if rtime != None:
-            os.utime(os.path.join(tmpdir, tmpmessagename), (rtime, rtime))
-        self.ui.debug('maildir', 'savemessage: moving from %s to %s' % \
-                 (tmpmessagename, messagename))
-        if tmpmessagename != messagename: # then rename it
-            os.rename(os.path.join(tmpdir, tmpmessagename),
-                    os.path.join(tmpdir, messagename))
-
-        if self.dofsync:
-            try:
-                # fsync the directory (safer semantics in Linux)
-                fd = os.open(tmpdir, os.O_RDONLY)
-                os.fsync(fd)
-                os.close(fd)
-            except:
-                pass
+            os.utime(os.path.join(tmpdir, messagename), (rtime, rtime))
 
         self.messagelist[uid] = {'uid': uid, 'flags': [],
                                  'filename': os.path.join('tmp', messagename)}