tobias/docker-offlineimap
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

/offlineimap/head: changeset 347

Browse Source 
Preparing for 3.99.7
This commit is contained in:
jgoerzen 2003-01-09 03:16:07 +01:00
parent 399f7d9de9
commit 3cb8fb0062
7 changed files with 147 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
offlineimap/head/debian/changelog
View File

@ -1,3 +1,33 @@
offlineimap (3.99.7) unstable; urgency=low
* This is a 4.0 TRACK release, and may be unstable or in flux!
* Converted entire manual to DocBook SGML so it will be easier to
expand in the future. The HTML manual, also, looks far nicer now
than it did before.
* Fixed the Tk.VerboseUI -- small, silly error introduced in 3.99.6.
* Multiple performance and reliability enhancements to syncing
algorithms, as described below.
* The process of uploading new messages from local folders to the IMAP
server was not internally multi-threaded previously. Now it is.
This means that if you have a single folder with lots of new messages
locally, the processing time should be dramatically sped up. Moreover,
the process should be more reliable because we do not risk connections
going dead.
* The process of synchronizing flags has been overhauled and optimized.
Previously, for each message where a flag (seen, replied, etc.) was
changed, we'd issue a separate command to the IMAP server to adjust
things. Now, we issue one command for each flag. In other words,
instead of seing 45 messages saying something like "Adding flag S to
message 1421", you now see one message saying "Adding flag S to 45
messages" -- and the interaction with the IMAP server may well be
almost 45 times faster on this. We will now issue at most four
commands per flag operation (add or remove) per folder, where before,
we may have issued as many as two per message. Should be a
large speedup in most cases, but a small slowdown in a few.
* Potentially increased the reliability of writing files to the Maildir.
-- John Goerzen <jgoerzen@complete.org> Wed, 8 Jan 2003 07:05:01 -0600
offlineimap (3.99.6) unstable; urgency=low
* This a 4.0 TRACK release, and may be unstable or in flux!

2
offlineimap/head/debian/control
View File

@ -2,7 +2,7 @@ Source: offlineimap
Section: mail
Priority: optional
Maintainer: John Goerzen <jgoerzen@complete.org>
Build-Depends-Indep: debhelper (>> 3.0.0), python2.2 (>= 2.2.1-4), python2.2-dev (>= 2.2.1-4), groff
Build-Depends-Indep: debhelper (>> 3.0.0), python2.2 (>= 2.2.1-4), python2.2-dev (>= 2.2.1-4), groff, docbook-utils
Standards-Version: 3.5.2
Package: offlineimap

118
offlineimap/head/offlineimap/folder/Base.py
View File

@ -131,7 +131,7 @@ class BaseFolder:
def addmessagesflags(self, uidlist, flags):
for uid in uidlist:
self.addmessageflags(uid)
self.addmessageflags(uid, flags)
def deletemessageflags(self, uid, flags):
"""Removes each flag given from the message's flag set. If a given
@ -143,6 +143,10 @@ class BaseFolder:
newflags.sort()
self.savemessageflags(uid, newflags)
def deletemessagesflags(self, uidlist, flags):
for uid in uidlist:
self.deletemessageflags(uid, flags)
def deletemessage(self, uid):
raise NotImplementedException
@ -150,6 +154,35 @@ class BaseFolder:
for uid in uidlist:
self.deletemessage(uid)
def syncmessagesto_neguid_msg(self, uid, dest, applyto, register = 1):
if register:
UIBase.getglobalui().registerthread(self.getaccountname())
UIBase.getglobalui().copyingmessage(uid, self, applyto)
successobject = None
successuid = None
message = self.getmessage(uid)
flags = self.getmessageflags(uid)
for tryappend in applyto:
successuid = tryappend.savemessage(uid, message, flags)
if successuid >= 0:
successobject = tryappend
break
# Did we succeed?
if successobject != None:
if successuid: # Only if IMAP actually assigned a UID
# Copy the message to the other remote servers.
for appendserver in \
[x for x in applyto if x != successobject]:
appendserver.savemessage(successuid, message, flags)
# Copy to its new name on the local server and delete
# the one without a UID.
self.savemessage(successuid, message, flags)
self.deletemessage(uid) # It'll be re-downloaded.
else:
# Did not find any server to take this message. Ignore.
pass
def syncmessagesto_neguid(self, dest, applyto):
"""Pass 1 of folder synchronization.
@ -158,33 +191,28 @@ class BaseFolder:
and once that succeeds, get the UID, add it to the others for real,
add it to local for real, and delete the fake one."""
for uid in self.getmessagelist().keys():
if uid >= 0:
continue
UIBase.getglobalui().copyingmessage(uid, self, applyto)
successobject = None
successuid = None
message = self.getmessage(uid)
flags = self.getmessageflags(uid)
for tryappend in applyto:
successuid = tryappend.savemessage(uid, message, flags)
if successuid >= 0:
successobject = tryappend
break
# Did we succeed?
if successobject != None:
if successuid: # Only if IMAP actually assigned a UID
# Copy the message to the other remote servers.
for appendserver in \
[x for x in applyto if x != successobject]:
appendserver.savemessage(successuid, message, flags)
# Copy to its new name on the local server and delete
# the one without a UID.
self.savemessage(successuid, message, flags)
self.deletemessage(uid) # It'll be re-downloaded.
uidlist = [uid for uid in self.getmessagelist().keys() if uid < 0]
threads = []
usethread = None
if applyto != None:
usethread = applyto[0]
for uid in uidlist:
if usethread:
usethread.waitforthread()
thread = InstanceLimitedThread(\
usethread.getcopyinstancelimit(),
target = self.syncmessagesto_neguid_msg,
name = "New msg sync from %s" % self.getvisiblename(),
args = (uid, dest, applyto))
thread.setDaemon(1)
thread.start()
threads.append(thread)
else:
# Did not find any server to take this message. Ignore.
pass
self.syncmessagesto_neguid_msg(uid, dest, applyto, register = 0)
for thread in threads:
thread.join()
def copymessageto(self, uid, applyto, register = 1):
# Sometimes, it could be the case that if a sync takes awhile,
@ -260,6 +288,17 @@ class BaseFolder:
Look for any flag matching issues -- set dest message to have the
same flags that we have."""
# As an optimization over previous versions, we store up which flags
# are being used for an add or a delete. For each flag, we store
# a list of uids to which it should be added. Then, we can call
# addmessagesflags() to apply them in bulk, rather than one
# call per message as before. This should result in some significant
# performance improvements.
addflaglist = {}
delflaglist = {}
for uid in self.getmessagelist().keys():
if uid < 0: # Ignore messages missed by pass 1
continue
@ -267,17 +306,26 @@ class BaseFolder:
destflags = dest.getmessageflags(uid)
addflags = [x for x in selfflags if x not in destflags]
if len(addflags):
UIBase.getglobalui().addingflags(uid, addflags, applyto)
for object in applyto:
object.addmessageflags(uid, addflags)
for flag in addflags:
if not flag in addflaglist:
addflaglist[flag] = []
addflaglist[flag].append(uid)
delflags = [x for x in destflags if x not in selfflags]
if len(delflags):
UIBase.getglobalui().deletingflags(uid, delflags, applyto)
for object in applyto:
object.deletemessageflags(uid, delflags)
for flag in delflags:
if not flag in delflaglist:
delflaglist[flag] = []
delflaglist[flag].append(uid)
for object in applyto:
for flag in addflaglist.keys():
UIBase.getglobalui().addingflags(addflaglist[flag], flag, [object])
object.addmessagesflags(addflaglist[flag], [flag])
for flag in delflaglist.keys():
UIBase.getglobalui().deletingflags(addflaglist[flag], flag, [object])
object.deletemessagesflags(delflaglist[flag], [flag])
def syncmessagesto(self, dest, applyto = None):
"""Syncs messages in this folder to the destination.
If applyto is specified, it should be a list of folders (don't forget

24
offlineimap/head/offlineimap/folder/IMAP.py
View File

@ -198,6 +198,15 @@ class IMAPFolder(BaseFolder):
self.addmessagesflags([uid], flags)
def addmessagesflags(self, uidlist, flags):
self.processmessagesflags('+', uidlist, flags)
def deletemessageflags(self, uid, flags):
self.deletemessagesflags([uid], flags)
def deletemessagesflags(self, uidlist, flags):
self.processmessagesflags('-', uidlist, flags)
def processmessagesflags(self, operation, uidlist, flags):
imapobj = self.imapserver.acquireconnection()
try:
try:
@ -207,7 +216,7 @@ class IMAPFolder(BaseFolder):
return
r = imapobj.uid('store',
imaputil.listjoin(uidlist),
'+FLAGS',
operation + 'FLAGS',
imaputil.flagsmaildir2imap(flags))
assert r[0] == 'OK', 'Error with store: ' + r[1]
r = r[1]
@ -234,10 +243,15 @@ class IMAPFolder(BaseFolder):
except ValueError: # Let it slide if it's not in the list
pass
for uid in needupdate:
for flag in flags:
if not flag in self.messagelist[uid]['flags']:
self.messagelist[uid]['flags'].append(flag)
self.messagelist[uid]['flags'].sort()
if operation == '+':
for flag in flags:
if not flag in self.messagelist[uid]['flags']:
self.messagelist[uid]['flags'].append(flag)
self.messagelist[uid]['flags'].sort()
elif operation == '-':
for flag in flags:
if flag in self.messagelist[uid]['flags']:
self.messagelist[uid]['flags'].remove(flag)
def deletemessage(self, uid):
self.deletemessages([uid])

7
offlineimap/head/offlineimap/folder/Maildir.py
View File

@ -170,12 +170,13 @@ class MaildirFolder(BaseFolder):
attempts += 1
else:
break
file = open(os.path.join(tmpdir, messagename), "wt")
tmpmessagename = messagename.split(',')[0]
file = open(os.path.join(tmpdir, tmpmessagename), "wt")
file.write(content)
file.close()
os.link(os.path.join(tmpdir, messagename),
os.link(os.path.join(tmpdir, tmpmessagename),
os.path.join(newdir, messagename))
os.unlink(os.path.join(tmpdir, messagename))
os.unlink(os.path.join(tmpdir, tmpmessagename))
self.messagelist[uid] = {'uid': uid, 'flags': [],
'filename': os.path.join(newdir, messagename)}
self.savemessageflags(uid, flags)

8
offlineimap/head/offlineimap/ui/Blinkenlights.py
View File

@ -63,13 +63,13 @@ class BlinkenBase:
s.gettf().setcolor('red')
s.__class__.__bases__[-1].deletingmessage(s, uid, destlist)
def addingflags(s, uid, flags, destlist):
def addingflags(s, uidlist, flags, destlist):
s.gettf().setcolor('yellow')
s.__class__.__bases__[-1].addingflags(s, uid, flags, destlist)
s.__class__.__bases__[-1].addingflags(s, uidlist, flags, destlist)
def deletingflags(s, uid, flags, destlist):
def deletingflags(s, uidlist, flags, destlist):
s.gettf().setcolor('pink')
s.__class__.__bases__[-1].deletingflags(s, uid, flags, destlist)
s.__class__.__bases__[-1].deletingflags(s, uidlist, flags, destlist)
def init_banner(s):
s.availablethreadframes = {}

12
offlineimap/head/offlineimap/ui/UIBase.py
View File

@ -221,17 +221,17 @@ class UIBase:
", ".join([str(u) for u in uidlist]),
ds))
def addingflags(s, uid, flags, destlist):
def addingflags(s, uidlist, flags, destlist):
if s.verbose >= 0:
ds = s.folderlist(destlist)
s._msg("Adding flags %s to message %d on %s" % \
(", ".join(flags), uid, ds))
s._msg("Adding flags %s to %d messages on %s" % \
(", ".join(flags), len(uidlist), ds))
def deletingflags(s, uid, flags, destlist):
def deletingflags(s, uidlist, flags, destlist):
if s.verbose >= 0:
ds = s.folderlist(destlist)
s._msg("Deleting flags %s to message %d on %s" % \
(", ".join(flags), uid, ds))
s._msg("Deleting flags %s to %d messages on %s" % \
(", ".join(flags), len(uidlist), ds))
################################################## Threads