/offlineimap/head: changeset 347

Preparing for 3.99.7

offlineimap/head/debian/changelog

@@ -1,3 +1,33 @@
+offlineimap (3.99.7) unstable; urgency=low
+
+  * This is a 4.0 TRACK release, and may be unstable or in flux!
+  * Converted entire manual to DocBook SGML so it will be easier to
+    expand in the future.  The HTML manual, also, looks far nicer now
+    than it did before.
+  * Fixed the Tk.VerboseUI -- small, silly error introduced in 3.99.6.
+  * Multiple performance and reliability enhancements to syncing
+    algorithms, as described below.
+  * The process of uploading new messages from local folders to the IMAP
+    server was not internally multi-threaded previously.  Now it is.
+    This means that if you have a single folder with lots of new messages
+    locally, the processing time should be dramatically sped up.  Moreover,
+    the process should be more reliable because we do not risk connections
+    going dead.
+  * The process of synchronizing flags has been overhauled and optimized.
+    Previously, for each message where a flag (seen, replied, etc.) was
+    changed, we'd issue a separate command to the IMAP server to adjust
+    things.  Now, we issue one command for each flag.  In other words,
+    instead of seing 45 messages saying something like "Adding flag S to
+    message 1421", you now see one message saying "Adding flag S to 45 
+    messages" -- and the interaction with the IMAP server may well be
+    almost 45 times faster on this.  We will now issue at most four
+    commands per flag operation (add or remove) per folder, where before,
+    we may have issued as many as two per message.  Should be a
+    large speedup in most cases, but a small slowdown in a few.
+  * Potentially increased the reliability of writing files to the Maildir.
+
+ -- John Goerzen <jgoerzen@complete.org>  Wed,  8 Jan 2003 07:05:01 -0600
+
 offlineimap (3.99.6) unstable; urgency=low
 
   * This a 4.0 TRACK release, and may be unstable or in flux!

offlineimap/head/debian/control

@@ -2,7 +2,7 @@ Source: offlineimap
 Section: mail
 Priority: optional
 Maintainer: John Goerzen <jgoerzen@complete.org>
-Build-Depends-Indep: debhelper (>> 3.0.0), python2.2 (>= 2.2.1-4), python2.2-dev (>= 2.2.1-4), groff
+Build-Depends-Indep: debhelper (>> 3.0.0), python2.2 (>= 2.2.1-4), python2.2-dev (>= 2.2.1-4), groff, docbook-utils
 Standards-Version: 3.5.2
 
 Package: offlineimap

offlineimap/head/offlineimap/folder/Base.py

@@ -131,7 +131,7 @@ class BaseFolder:
 
     def addmessagesflags(self, uidlist, flags):
         for uid in uidlist:
-            self.addmessageflags(uid)
+            self.addmessageflags(uid, flags)
 
     def deletemessageflags(self, uid, flags):
         """Removes each flag given from the message's flag set.  If a given
@@ -143,6 +143,10 @@ class BaseFolder:
         newflags.sort()
         self.savemessageflags(uid, newflags)
 
+    def deletemessagesflags(self, uidlist, flags):
+        for uid in uidlist:
+            self.deletemessageflags(uid, flags)
+
     def deletemessage(self, uid):
         raise NotImplementedException
 
@@ -150,6 +154,35 @@ class BaseFolder:
         for uid in uidlist:
             self.deletemessage(uid)
 
+    def syncmessagesto_neguid_msg(self, uid, dest, applyto, register = 1):
+        if register:
+            UIBase.getglobalui().registerthread(self.getaccountname())
+        UIBase.getglobalui().copyingmessage(uid, self, applyto)
+        successobject = None
+        successuid = None
+        message = self.getmessage(uid)
+        flags = self.getmessageflags(uid)
+        for tryappend in applyto:
+            successuid = tryappend.savemessage(uid, message, flags)
+            if successuid >= 0:
+                successobject = tryappend
+                break
+        # Did we succeed?
+        if successobject != None:
+            if successuid:       # Only if IMAP actually assigned a UID
+                # Copy the message to the other remote servers.
+                for appendserver in \
+                        [x for x in applyto if x != successobject]:
+                    appendserver.savemessage(successuid, message, flags)
+                    # Copy to its new name on the local server and delete
+                    # the one without a UID.
+                    self.savemessage(successuid, message, flags)
+            self.deletemessage(uid) # It'll be re-downloaded.
+        else:
+            # Did not find any server to take this message.  Ignore.
+            pass
+        
+
     def syncmessagesto_neguid(self, dest, applyto):
         """Pass 1 of folder synchronization.
 
@@ -158,33 +191,28 @@ class BaseFolder:
         and once that succeeds, get the UID, add it to the others for real,
         add it to local for real, and delete the fake one."""
 
-        for uid in self.getmessagelist().keys():
+        uidlist = [uid for uid in self.getmessagelist().keys() if uid < 0]
-            if uid >= 0:
+        threads = []
-                continue
+
-            UIBase.getglobalui().copyingmessage(uid, self, applyto)
+        usethread = None
-            successobject = None
+        if applyto != None:
-            successuid = None
+            usethread = applyto[0]
-            message = self.getmessage(uid)
+        
-            flags = self.getmessageflags(uid)
+        for uid in uidlist:
-            for tryappend in applyto:
+            if usethread:
-                successuid = tryappend.savemessage(uid, message, flags)
+                usethread.waitforthread()
-                if successuid >= 0:
+                thread = InstanceLimitedThread(\
-                    successobject = tryappend
+                    usethread.getcopyinstancelimit(),
-                    break
+                    target = self.syncmessagesto_neguid_msg,
-            # Did we succeed?
+                    name = "New msg sync from %s" % self.getvisiblename(),
-            if successobject != None:
+                    args = (uid, dest, applyto))
-                if successuid:       # Only if IMAP actually assigned a UID
+                thread.setDaemon(1)
-                    # Copy the message to the other remote servers.
+                thread.start()
-                    for appendserver in \
+                threads.append(thread)
-                            [x for x in applyto if x != successobject]:
-                        appendserver.savemessage(successuid, message, flags)
-                        # Copy to its new name on the local server and delete
-                        # the one without a UID.
-                        self.savemessage(successuid, message, flags)
-                self.deletemessage(uid) # It'll be re-downloaded.
             else:
-                # Did not find any server to take this message.  Ignore.
+                self.syncmessagesto_neguid_msg(uid, dest, applyto, register = 0)
-                pass
+        for thread in threads:
+            thread.join()
 
     def copymessageto(self, uid, applyto, register = 1):
         # Sometimes, it could be the case that if a sync takes awhile,
@@ -260,6 +288,17 @@ class BaseFolder:
 
         Look for any flag matching issues -- set dest message to have the
         same flags that we have."""
+
+        # As an optimization over previous versions, we store up which flags
+        # are being used for an add or a delete.  For each flag, we store
+        # a list of uids to which it should be added.  Then, we can call
+        # addmessagesflags() to apply them in bulk, rather than one
+        # call per message as before.  This should result in some significant
+        # performance improvements.
+
+        addflaglist = {}
+        delflaglist = {}
+        
         for uid in self.getmessagelist().keys():
             if uid < 0:                 # Ignore messages missed by pass 1
                 continue
@@ -267,17 +306,26 @@ class BaseFolder:
             destflags = dest.getmessageflags(uid)
 
             addflags = [x for x in selfflags if x not in destflags]
-            if len(addflags):
+
-                UIBase.getglobalui().addingflags(uid, addflags, applyto)
+            for flag in addflags:
-                for object in applyto:
+                if not flag in addflaglist:
-                    object.addmessageflags(uid, addflags)
+                    addflaglist[flag] = []
+                addflaglist[flag].append(uid)
 
             delflags = [x for x in destflags if x not in selfflags]
-            if len(delflags):
+            for flag in delflags:
-                UIBase.getglobalui().deletingflags(uid, delflags, applyto)
+                if not flag in delflaglist:
-                for object in applyto:
+                    delflaglist[flag] = []
-                    object.deletemessageflags(uid, delflags)
+                delflaglist[flag].append(uid)
 
+        for object in applyto:
+            for flag in addflaglist.keys():
+                UIBase.getglobalui().addingflags(addflaglist[flag], flag, [object])
+                object.addmessagesflags(addflaglist[flag], [flag])
+            for flag in delflaglist.keys():
+                UIBase.getglobalui().deletingflags(addflaglist[flag], flag, [object])
+                object.deletemessagesflags(delflaglist[flag], [flag])
+                
     def syncmessagesto(self, dest, applyto = None):
         """Syncs messages in this folder to the destination.
         If applyto is specified, it should be a list of folders (don't forget

offlineimap/head/offlineimap/folder/IMAP.py

@@ -198,6 +198,15 @@ class IMAPFolder(BaseFolder):
         self.addmessagesflags([uid], flags)
 
     def addmessagesflags(self, uidlist, flags):
+        self.processmessagesflags('+', uidlist, flags)
+
+    def deletemessageflags(self, uid, flags):
+        self.deletemessagesflags([uid], flags)
+
+    def deletemessagesflags(self, uidlist, flags):
+        self.processmessagesflags('-', uidlist, flags)
+
+    def processmessagesflags(self, operation, uidlist, flags):
         imapobj = self.imapserver.acquireconnection()
         try:
             try:
@@ -207,7 +216,7 @@ class IMAPFolder(BaseFolder):
                 return
             r = imapobj.uid('store',
                             imaputil.listjoin(uidlist),
-                            '+FLAGS',
+                            operation + 'FLAGS',
                             imaputil.flagsmaildir2imap(flags))
             assert r[0] == 'OK', 'Error with store: ' + r[1]
             r = r[1]
@@ -234,10 +243,15 @@ class IMAPFolder(BaseFolder):
             except ValueError:          # Let it slide if it's not in the list
                 pass
         for uid in needupdate:
-            for flag in flags:
+            if operation == '+':
-                if not flag in self.messagelist[uid]['flags']:
+                for flag in flags:
-                    self.messagelist[uid]['flags'].append(flag)
+                    if not flag in self.messagelist[uid]['flags']:
-                self.messagelist[uid]['flags'].sort()
+                        self.messagelist[uid]['flags'].append(flag)
+                    self.messagelist[uid]['flags'].sort()
+            elif operation == '-':
+                for flag in flags:
+                    if flag in self.messagelist[uid]['flags']:
+                        self.messagelist[uid]['flags'].remove(flag)
 
     def deletemessage(self, uid):
         self.deletemessages([uid])

offlineimap/head/offlineimap/folder/Maildir.py

@@ -170,12 +170,13 @@ class MaildirFolder(BaseFolder):
                 attempts += 1
             else:
                 break
-        file = open(os.path.join(tmpdir, messagename), "wt")
+        tmpmessagename = messagename.split(',')[0]
+        file = open(os.path.join(tmpdir, tmpmessagename), "wt")
         file.write(content)
         file.close()
-        os.link(os.path.join(tmpdir, messagename),
+        os.link(os.path.join(tmpdir, tmpmessagename),
                 os.path.join(newdir, messagename))
-        os.unlink(os.path.join(tmpdir, messagename))
+        os.unlink(os.path.join(tmpdir, tmpmessagename))
         self.messagelist[uid] = {'uid': uid, 'flags': [],
                                  'filename': os.path.join(newdir, messagename)}
         self.savemessageflags(uid, flags)

offlineimap/head/offlineimap/ui/Blinkenlights.py

@@ -63,13 +63,13 @@ class BlinkenBase:
         s.gettf().setcolor('red')
         s.__class__.__bases__[-1].deletingmessage(s, uid, destlist)
 
-    def addingflags(s, uid, flags, destlist):
+    def addingflags(s, uidlist, flags, destlist):
         s.gettf().setcolor('yellow')
-        s.__class__.__bases__[-1].addingflags(s, uid, flags, destlist)
+        s.__class__.__bases__[-1].addingflags(s, uidlist, flags, destlist)
 
-    def deletingflags(s, uid, flags, destlist):
+    def deletingflags(s, uidlist, flags, destlist):
         s.gettf().setcolor('pink')
-        s.__class__.__bases__[-1].deletingflags(s, uid, flags, destlist)
+        s.__class__.__bases__[-1].deletingflags(s, uidlist, flags, destlist)
 
     def init_banner(s):
         s.availablethreadframes = {}

offlineimap/head/offlineimap/ui/UIBase.py

@@ -221,17 +221,17 @@ class UIBase:
                     ", ".join([str(u) for u in uidlist]),
                     ds))
 
-    def addingflags(s, uid, flags, destlist):
+    def addingflags(s, uidlist, flags, destlist):
         if s.verbose >= 0:
             ds = s.folderlist(destlist)
-            s._msg("Adding flags %s to message %d on %s" % \
+            s._msg("Adding flags %s to %d messages  on %s" % \
-                   (", ".join(flags), uid, ds))
+                   (", ".join(flags), len(uidlist), ds))
 
-    def deletingflags(s, uid, flags, destlist):
+    def deletingflags(s, uidlist, flags, destlist):
         if s.verbose >= 0:
             ds = s.folderlist(destlist)
-            s._msg("Deleting flags %s to message %d on %s" % \
+            s._msg("Deleting flags %s to %d messages on %s" % \
-                   (", ".join(flags), uid, ds))
+                   (", ".join(flags), len(uidlist), ds))
 
     ################################################## Threads