From 3cb8fb0062ef700dbc9e83732510fa9fa1f0c09e Mon Sep 17 00:00:00 2001 From: jgoerzen Date: Thu, 9 Jan 2003 03:16:07 +0100 Subject: [PATCH] /offlineimap/head: changeset 347 Preparing for 3.99.7 --- offlineimap/head/debian/changelog | 30 +++++ offlineimap/head/debian/control | 2 +- offlineimap/head/offlineimap/folder/Base.py | 118 ++++++++++++------ offlineimap/head/offlineimap/folder/IMAP.py | 24 +++- .../head/offlineimap/folder/Maildir.py | 7 +- .../head/offlineimap/ui/Blinkenlights.py | 8 +- offlineimap/head/offlineimap/ui/UIBase.py | 12 +- 7 files changed, 147 insertions(+), 54 deletions(-) diff --git a/offlineimap/head/debian/changelog b/offlineimap/head/debian/changelog index d761360..0d1ee89 100644 --- a/offlineimap/head/debian/changelog +++ b/offlineimap/head/debian/changelog @@ -1,3 +1,33 @@ +offlineimap (3.99.7) unstable; urgency=low + + * This is a 4.0 TRACK release, and may be unstable or in flux! + * Converted entire manual to DocBook SGML so it will be easier to + expand in the future. The HTML manual, also, looks far nicer now + than it did before. + * Fixed the Tk.VerboseUI -- small, silly error introduced in 3.99.6. + * Multiple performance and reliability enhancements to syncing + algorithms, as described below. + * The process of uploading new messages from local folders to the IMAP + server was not internally multi-threaded previously. Now it is. + This means that if you have a single folder with lots of new messages + locally, the processing time should be dramatically sped up. Moreover, + the process should be more reliable because we do not risk connections + going dead. + * The process of synchronizing flags has been overhauled and optimized. + Previously, for each message where a flag (seen, replied, etc.) was + changed, we'd issue a separate command to the IMAP server to adjust + things. Now, we issue one command for each flag. In other words, + instead of seing 45 messages saying something like "Adding flag S to + message 1421", you now see one message saying "Adding flag S to 45 + messages" -- and the interaction with the IMAP server may well be + almost 45 times faster on this. We will now issue at most four + commands per flag operation (add or remove) per folder, where before, + we may have issued as many as two per message. Should be a + large speedup in most cases, but a small slowdown in a few. + * Potentially increased the reliability of writing files to the Maildir. + + -- John Goerzen Wed, 8 Jan 2003 07:05:01 -0600 + offlineimap (3.99.6) unstable; urgency=low * This a 4.0 TRACK release, and may be unstable or in flux! diff --git a/offlineimap/head/debian/control b/offlineimap/head/debian/control index b6250ec..0d87dfe 100644 --- a/offlineimap/head/debian/control +++ b/offlineimap/head/debian/control @@ -2,7 +2,7 @@ Source: offlineimap Section: mail Priority: optional Maintainer: John Goerzen -Build-Depends-Indep: debhelper (>> 3.0.0), python2.2 (>= 2.2.1-4), python2.2-dev (>= 2.2.1-4), groff +Build-Depends-Indep: debhelper (>> 3.0.0), python2.2 (>= 2.2.1-4), python2.2-dev (>= 2.2.1-4), groff, docbook-utils Standards-Version: 3.5.2 Package: offlineimap diff --git a/offlineimap/head/offlineimap/folder/Base.py b/offlineimap/head/offlineimap/folder/Base.py index 06d61a5..738b893 100644 --- a/offlineimap/head/offlineimap/folder/Base.py +++ b/offlineimap/head/offlineimap/folder/Base.py @@ -131,7 +131,7 @@ class BaseFolder: def addmessagesflags(self, uidlist, flags): for uid in uidlist: - self.addmessageflags(uid) + self.addmessageflags(uid, flags) def deletemessageflags(self, uid, flags): """Removes each flag given from the message's flag set. If a given @@ -143,6 +143,10 @@ class BaseFolder: newflags.sort() self.savemessageflags(uid, newflags) + def deletemessagesflags(self, uidlist, flags): + for uid in uidlist: + self.deletemessageflags(uid, flags) + def deletemessage(self, uid): raise NotImplementedException @@ -150,6 +154,35 @@ class BaseFolder: for uid in uidlist: self.deletemessage(uid) + def syncmessagesto_neguid_msg(self, uid, dest, applyto, register = 1): + if register: + UIBase.getglobalui().registerthread(self.getaccountname()) + UIBase.getglobalui().copyingmessage(uid, self, applyto) + successobject = None + successuid = None + message = self.getmessage(uid) + flags = self.getmessageflags(uid) + for tryappend in applyto: + successuid = tryappend.savemessage(uid, message, flags) + if successuid >= 0: + successobject = tryappend + break + # Did we succeed? + if successobject != None: + if successuid: # Only if IMAP actually assigned a UID + # Copy the message to the other remote servers. + for appendserver in \ + [x for x in applyto if x != successobject]: + appendserver.savemessage(successuid, message, flags) + # Copy to its new name on the local server and delete + # the one without a UID. + self.savemessage(successuid, message, flags) + self.deletemessage(uid) # It'll be re-downloaded. + else: + # Did not find any server to take this message. Ignore. + pass + + def syncmessagesto_neguid(self, dest, applyto): """Pass 1 of folder synchronization. @@ -158,33 +191,28 @@ class BaseFolder: and once that succeeds, get the UID, add it to the others for real, add it to local for real, and delete the fake one.""" - for uid in self.getmessagelist().keys(): - if uid >= 0: - continue - UIBase.getglobalui().copyingmessage(uid, self, applyto) - successobject = None - successuid = None - message = self.getmessage(uid) - flags = self.getmessageflags(uid) - for tryappend in applyto: - successuid = tryappend.savemessage(uid, message, flags) - if successuid >= 0: - successobject = tryappend - break - # Did we succeed? - if successobject != None: - if successuid: # Only if IMAP actually assigned a UID - # Copy the message to the other remote servers. - for appendserver in \ - [x for x in applyto if x != successobject]: - appendserver.savemessage(successuid, message, flags) - # Copy to its new name on the local server and delete - # the one without a UID. - self.savemessage(successuid, message, flags) - self.deletemessage(uid) # It'll be re-downloaded. + uidlist = [uid for uid in self.getmessagelist().keys() if uid < 0] + threads = [] + + usethread = None + if applyto != None: + usethread = applyto[0] + + for uid in uidlist: + if usethread: + usethread.waitforthread() + thread = InstanceLimitedThread(\ + usethread.getcopyinstancelimit(), + target = self.syncmessagesto_neguid_msg, + name = "New msg sync from %s" % self.getvisiblename(), + args = (uid, dest, applyto)) + thread.setDaemon(1) + thread.start() + threads.append(thread) else: - # Did not find any server to take this message. Ignore. - pass + self.syncmessagesto_neguid_msg(uid, dest, applyto, register = 0) + for thread in threads: + thread.join() def copymessageto(self, uid, applyto, register = 1): # Sometimes, it could be the case that if a sync takes awhile, @@ -260,6 +288,17 @@ class BaseFolder: Look for any flag matching issues -- set dest message to have the same flags that we have.""" + + # As an optimization over previous versions, we store up which flags + # are being used for an add or a delete. For each flag, we store + # a list of uids to which it should be added. Then, we can call + # addmessagesflags() to apply them in bulk, rather than one + # call per message as before. This should result in some significant + # performance improvements. + + addflaglist = {} + delflaglist = {} + for uid in self.getmessagelist().keys(): if uid < 0: # Ignore messages missed by pass 1 continue @@ -267,17 +306,26 @@ class BaseFolder: destflags = dest.getmessageflags(uid) addflags = [x for x in selfflags if x not in destflags] - if len(addflags): - UIBase.getglobalui().addingflags(uid, addflags, applyto) - for object in applyto: - object.addmessageflags(uid, addflags) + + for flag in addflags: + if not flag in addflaglist: + addflaglist[flag] = [] + addflaglist[flag].append(uid) delflags = [x for x in destflags if x not in selfflags] - if len(delflags): - UIBase.getglobalui().deletingflags(uid, delflags, applyto) - for object in applyto: - object.deletemessageflags(uid, delflags) + for flag in delflags: + if not flag in delflaglist: + delflaglist[flag] = [] + delflaglist[flag].append(uid) + for object in applyto: + for flag in addflaglist.keys(): + UIBase.getglobalui().addingflags(addflaglist[flag], flag, [object]) + object.addmessagesflags(addflaglist[flag], [flag]) + for flag in delflaglist.keys(): + UIBase.getglobalui().deletingflags(addflaglist[flag], flag, [object]) + object.deletemessagesflags(delflaglist[flag], [flag]) + def syncmessagesto(self, dest, applyto = None): """Syncs messages in this folder to the destination. If applyto is specified, it should be a list of folders (don't forget diff --git a/offlineimap/head/offlineimap/folder/IMAP.py b/offlineimap/head/offlineimap/folder/IMAP.py index 87a8644..9bb7785 100644 --- a/offlineimap/head/offlineimap/folder/IMAP.py +++ b/offlineimap/head/offlineimap/folder/IMAP.py @@ -198,6 +198,15 @@ class IMAPFolder(BaseFolder): self.addmessagesflags([uid], flags) def addmessagesflags(self, uidlist, flags): + self.processmessagesflags('+', uidlist, flags) + + def deletemessageflags(self, uid, flags): + self.deletemessagesflags([uid], flags) + + def deletemessagesflags(self, uidlist, flags): + self.processmessagesflags('-', uidlist, flags) + + def processmessagesflags(self, operation, uidlist, flags): imapobj = self.imapserver.acquireconnection() try: try: @@ -207,7 +216,7 @@ class IMAPFolder(BaseFolder): return r = imapobj.uid('store', imaputil.listjoin(uidlist), - '+FLAGS', + operation + 'FLAGS', imaputil.flagsmaildir2imap(flags)) assert r[0] == 'OK', 'Error with store: ' + r[1] r = r[1] @@ -234,10 +243,15 @@ class IMAPFolder(BaseFolder): except ValueError: # Let it slide if it's not in the list pass for uid in needupdate: - for flag in flags: - if not flag in self.messagelist[uid]['flags']: - self.messagelist[uid]['flags'].append(flag) - self.messagelist[uid]['flags'].sort() + if operation == '+': + for flag in flags: + if not flag in self.messagelist[uid]['flags']: + self.messagelist[uid]['flags'].append(flag) + self.messagelist[uid]['flags'].sort() + elif operation == '-': + for flag in flags: + if flag in self.messagelist[uid]['flags']: + self.messagelist[uid]['flags'].remove(flag) def deletemessage(self, uid): self.deletemessages([uid]) diff --git a/offlineimap/head/offlineimap/folder/Maildir.py b/offlineimap/head/offlineimap/folder/Maildir.py index 468e226..5fa2f2a 100644 --- a/offlineimap/head/offlineimap/folder/Maildir.py +++ b/offlineimap/head/offlineimap/folder/Maildir.py @@ -170,12 +170,13 @@ class MaildirFolder(BaseFolder): attempts += 1 else: break - file = open(os.path.join(tmpdir, messagename), "wt") + tmpmessagename = messagename.split(',')[0] + file = open(os.path.join(tmpdir, tmpmessagename), "wt") file.write(content) file.close() - os.link(os.path.join(tmpdir, messagename), + os.link(os.path.join(tmpdir, tmpmessagename), os.path.join(newdir, messagename)) - os.unlink(os.path.join(tmpdir, messagename)) + os.unlink(os.path.join(tmpdir, tmpmessagename)) self.messagelist[uid] = {'uid': uid, 'flags': [], 'filename': os.path.join(newdir, messagename)} self.savemessageflags(uid, flags) diff --git a/offlineimap/head/offlineimap/ui/Blinkenlights.py b/offlineimap/head/offlineimap/ui/Blinkenlights.py index 6ed26bc..60c3704 100644 --- a/offlineimap/head/offlineimap/ui/Blinkenlights.py +++ b/offlineimap/head/offlineimap/ui/Blinkenlights.py @@ -63,13 +63,13 @@ class BlinkenBase: s.gettf().setcolor('red') s.__class__.__bases__[-1].deletingmessage(s, uid, destlist) - def addingflags(s, uid, flags, destlist): + def addingflags(s, uidlist, flags, destlist): s.gettf().setcolor('yellow') - s.__class__.__bases__[-1].addingflags(s, uid, flags, destlist) + s.__class__.__bases__[-1].addingflags(s, uidlist, flags, destlist) - def deletingflags(s, uid, flags, destlist): + def deletingflags(s, uidlist, flags, destlist): s.gettf().setcolor('pink') - s.__class__.__bases__[-1].deletingflags(s, uid, flags, destlist) + s.__class__.__bases__[-1].deletingflags(s, uidlist, flags, destlist) def init_banner(s): s.availablethreadframes = {} diff --git a/offlineimap/head/offlineimap/ui/UIBase.py b/offlineimap/head/offlineimap/ui/UIBase.py index c27981c..776275a 100644 --- a/offlineimap/head/offlineimap/ui/UIBase.py +++ b/offlineimap/head/offlineimap/ui/UIBase.py @@ -221,17 +221,17 @@ class UIBase: ", ".join([str(u) for u in uidlist]), ds)) - def addingflags(s, uid, flags, destlist): + def addingflags(s, uidlist, flags, destlist): if s.verbose >= 0: ds = s.folderlist(destlist) - s._msg("Adding flags %s to message %d on %s" % \ - (", ".join(flags), uid, ds)) + s._msg("Adding flags %s to %d messages on %s" % \ + (", ".join(flags), len(uidlist), ds)) - def deletingflags(s, uid, flags, destlist): + def deletingflags(s, uidlist, flags, destlist): if s.verbose >= 0: ds = s.folderlist(destlist) - s._msg("Deleting flags %s to message %d on %s" % \ - (", ".join(flags), uid, ds)) + s._msg("Deleting flags %s to %d messages on %s" % \ + (", ".join(flags), len(uidlist), ds)) ################################################## Threads