From d75a7a862f2954af8fbaa49ceef4e92e8087e1ba Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 14:02:22 +0100 Subject: [PATCH 01/12] Fix regression of MachineUI The recent UI overhaul led to the breakage of the machineui. This bug fixes that. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 2 ++ offlineimap/ui/Machine.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 76a0ea3..5a113b2 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -18,3 +18,5 @@ Changes Bug Fixes --------- + +* Fix regression that broke MachineUI diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 868433a..0a51bcc 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -43,7 +43,7 @@ class MachineUI(UIBase): 'warn', '', currentThread().getName(), msg)) def registerthread(self, account): - super(MachineUI, self).registerthread(self, account) + super(MachineUI, self).registerthread(account) self._printData('registerthread', account) def unregisterthread(s, thread): From de4f8c8605f55388f2717e0989252dff01c3c352 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 14:06:03 +0100 Subject: [PATCH 02/12] Remove unneeded import in MachineUI The urllib module is not needed anymore. Signed-off-by: Sebastian Spaeth --- offlineimap/ui/Machine.py | 1 - 1 file changed, 1 deletion(-) diff --git a/offlineimap/ui/Machine.py b/offlineimap/ui/Machine.py index 0a51bcc..e6d1877 100644 --- a/offlineimap/ui/Machine.py +++ b/offlineimap/ui/Machine.py @@ -14,7 +14,6 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import urllib import sys import time import logging From 6fe808338c1826a66eb507830be023dc1db9978a Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 30 Aug 2011 11:01:49 +0200 Subject: [PATCH 03/12] Refactor parsing out maildirs filename components Create a helper function that retrieves the UID, folder MD5, and Flags from a message filename. We need these items when we simply want to rename (=new UID) a Maildir message file later. The new function can give us these components. Rework, so we cache the calculation of the folder's md5 value once, it never changes and we call it a lot. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Maildir.py | 116 +++++++++++++++++++--------------- 1 file changed, 66 insertions(+), 50 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index c9ea63d..dc2e368 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -1,6 +1,5 @@ # Maildir folder support -# Copyright (C) 2002 - 2007 John Goerzen -# +# Copyright (C) 2002 - 2011 John Goerzen & contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -35,8 +34,10 @@ except NameError: from offlineimap import OfflineImapError -uidmatchre = re.compile(',U=(\d+)') -timestampmatchre = re.compile('(\d+)'); +# Find the UID in a message filename +re_uidmatch = re.compile(',U=(\d+)') +# Find a numeric timestamp in a string (filename prefix) +re_timestampmatch = re.compile('(\d+)'); timeseq = 0 lasttime = long(0) @@ -67,11 +68,14 @@ class MaildirFolder(BaseFolder): # check if we should use a different infosep to support Win file systems self.wincompatible = self.config.getdefaultboolean( "Account "+self.accountname, "maildir-windows-compatible", False) - self.infosep = '!' if self.wincompatible else ':' """infosep is the separator between maildir name and flag appendix""" - self.flagmatchre = re.compile('(%s2,)(\w*)' % self.infosep) + self.re_flagmatch = re.compile('%s2,(\w*)' % self.infosep) #self.ui is set in BaseFolder.init() + # Everything up to the first comma or colon (or ! if Windows): + self.re_prefixmatch = re.compile('([^'+ self.infosep + ',]*)') + #folder's md, so we can match with recorded file md5 for validity + self._foldermd5 = md5(self.getvisiblename()).hexdigest() # Cache the full folder path, as we use getfullname() very often self._fullname = os.path.join(self.getroot(), self.getname()) @@ -97,7 +101,7 @@ class MaildirFolder(BaseFolder): + oldest_time_struct[5]) oldest_time_utc -= oldest_time_today_seconds - timestampmatch = timestampmatchre.search(messagename) + timestampmatch = re_timestampmatch.search(messagename) timestampstr = timestampmatch.group() timestamplong = long(timestampstr) if(timestamplong < oldest_time_utc): @@ -105,68 +109,80 @@ class MaildirFolder(BaseFolder): else: return True + def _parse_filename(self, filename): + """Returns a messages file name components + + Receives the file name (without path) of a msg. Usual format is + '<%d_%d.%d.%s>,U=<%d>,FMD5=<%s>:2,' (pointy brackets + denoting the various components). + + If FMD5 does not correspond with the current folder MD5, we will + return None for the UID & FMD5 (as it is not valid in this + folder). If UID or FMD5 can not be detected, we return `None` + for the respective element. If flags are empty or cannot be + detected, we return an empty flags list. + + :returns: (prefix, UID, FMD5, flags). UID is a numeric "long" + type. flags is a set() of Maildir flags""" + prefix, uid, fmd5, flags = None, None, None, set() + prefixmatch = self.re_prefixmatch.match(filename) + if prefixmatch: + prefix = prefixmatch.group(1) + folderstr = ',FMD5=%s' % self._foldermd5 + foldermatch = folderstr in filename + # If there was no folder MD5 specified, or if it mismatches, + # assume it is a foreign (new) message and ret: uid, fmd5 = None, None + if foldermatch: + uidmatch = re_uidmatch.search(filename) + if uidmatch: + uid = long(uidmatch.group(1)) + flagmatch = self.re_flagmatch.search(filename) + if flagmatch: + flags = set(flagmatch.group(1)) + return prefix, uid, fmd5, flags def _scanfolder(self): - """Cache the message list. Maildir flags are: - R (replied) - S (seen) - T (trashed) - D (draft) - F (flagged) - and must occur in ASCII order.""" + """Cache the message list from a Maildir. + + Maildir flags are: R (replied) S (seen) T (trashed) D (draft) F + (flagged). + :returns: dict that can be used as self.messagelist""" + maxage = self.config.getdefaultint("Account " + self.accountname, + "maxage", None) + maxsize = self.config.getdefaultint("Account " + self.accountname, + "maxsize", None) retval = {} files = [] - nouidcounter = -1 # Messages without UIDs get - # negative UID numbers. - foldermd5 = md5(self.getvisiblename()).hexdigest() - folderstr = ',FMD5=' + foldermd5 + nouidcounter = -1 # Messages without UIDs get negative UIDs. for dirannex in ['new', 'cur']: fulldirname = os.path.join(self.getfullname(), dirannex) - files.extend(os.path.join(dirannex, filename) for + files.extend((dirannex, filename) for filename in os.listdir(fulldirname)) - for file in files: - messagename = os.path.basename(file) - #check if there is a parameter for maxage / maxsize - then see if this - #message should be considered or not - maxage = self.config.getdefaultint("Account " + self.accountname, "maxage", -1) - maxsize = self.config.getdefaultint("Account " + self.accountname, "maxsize", -1) + for dirannex, filename in files: + # We store just dirannex and filename, ie 'cur/123...' + filepath = os.path.join(dirannex, filename) + # check maxage/maxsize if this message should be considered + if maxage and not self._iswithinmaxage(filename, maxage): + continue + if maxsize and (os.path.getsize(os.path.join( + self.getfullname(), filepath)) > maxsize): + continue - if(maxage != -1): - isnewenough = self._iswithinmaxage(messagename, maxage) - if(isnewenough != True): - #this message is older than we should consider.... - continue - - #Check and see if the message is too big if the maxsize for this account is set - if(maxsize != -1): - size = os.path.getsize(os.path.join(self.getfullname(), file)) - if(size > maxsize): - continue - - foldermatch = messagename.find(folderstr) != -1 - if not foldermatch: - # If there is no folder MD5 specified, or if it mismatches, - # assume it is a foreign (new) message and generate a - # negative uid for it + (prefix, uid, fmd5, flags) = self._parse_filename(filename) + if uid is None: # assign negative uid to upload it. uid = nouidcounter nouidcounter -= 1 else: # It comes from our folder. - uidmatch = uidmatchre.search(messagename) + uidmatch = re_uidmatch.search(filename) uid = None if not uidmatch: uid = nouidcounter nouidcounter -= 1 else: uid = long(uidmatch.group(1)) - #identify flags in the path name - flagmatch = self.flagmatchre.search(messagename) - if flagmatch: - flags = set(flagmatch.group(2)) - else: - flags = set() # 'filename' is 'dirannex/filename', e.g. cur/123,U=1,FMD5=1:2,S - retval[uid] = {'flags': flags, 'filename': file} + retval[uid] = {'flags': flags, 'filename': filepath} return retval def quickchanged(self, statusfolder): From 09ce56c5942db78193f1e44343759afb2d2d8396 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 30 Aug 2011 10:51:33 +0200 Subject: [PATCH 04/12] Factor out creating a Maildir message filename Various functions (such as change_message_uid) will want to construct maildir filenames, so factor out the code into a helper. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Maildir.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index dc2e368..5b2de91 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -221,6 +221,17 @@ class MaildirFolder(BaseFolder): filepath = os.path.join(self.getfullname(), filename) return os.path.getmtime(filepath) + def new_message_filename(self, uid, flags=set()): + """Creates a new unique Maildir filename + + :param uid: The UID`None`, or a set of maildir flags + :param flags: A set of maildir flags + :returns: String containing unique message filename""" + timeval, timeseq = gettimeseq() + return '%d_%d.%d.%s,U=%d,FMD5=%s%s2,%s' % \ + (timeval, timeseq, os.getpid(), socket.gethostname(), + uid, self._foldermd5, self.infosep, ''.join(sorted(flags))) + def savemessage(self, uid, content, flags, rtime): # This function only ever saves to tmp/, # but it calls savemessageflags() to actually save to cur/ or new/. @@ -237,14 +248,7 @@ class MaildirFolder(BaseFolder): # Otherwise, save the message in tmp/ and then call savemessageflags() # to give it a permanent home. tmpdir = os.path.join(self.getfullname(), 'tmp') - timeval, timeseq = gettimeseq() - messagename = '%d_%d.%d.%s,U=%d,FMD5=%s' % \ - (timeval, - timeseq, - os.getpid(), - socket.gethostname(), - uid, - md5(self.getvisiblename()).hexdigest()) + messagename = self.new_message_filename(uid, flags) # open file and write it out try: fd = os.open(os.path.join(tmpdir, messagename), From de537dc09c208afb1cbad4b024b2cec10f5e5123 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 30 Aug 2011 10:52:11 +0200 Subject: [PATCH 05/12] Implement change_message_uid Previously, assigning a new UID to a mapped IMAP or Maildir repository was done by loading the "local" item, saving it under a new UID and deleting the old one. This involved lots of disk activity for nothing more than an effective file rename in Maildirs, and lots of network usage in the MappedUID cases. We do this on every upload from a local to a remote item, so that can potentially be quite expensive. This patch lets backends that support it (Maildir, MappedUID) efficiently rename the file rather than having to read the mail content, write it out as a new file and delete the old file. This speeds up uploads from Maildir and the MappedUID server. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 4 +++ offlineimap/folder/Base.py | 39 ++++++++++++++------------ offlineimap/folder/IMAP.py | 10 ++++++- offlineimap/folder/Maildir.py | 52 +++++++++++++++++++++++------------ offlineimap/folder/UIDMaps.py | 25 +++++++++++++++++ 5 files changed, 93 insertions(+), 37 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 5a113b2..5fd29e4 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,6 +16,10 @@ New Features Changes ------- +* Uploading of Messages from Maildir and IMAP<->IMAP has been made more + efficient by renaming files/mapping entries, rather than actually + loading and saving the message under a new UID. + Bug Fixes --------- diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index f61e89d..0e065e3 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -234,6 +234,15 @@ class BaseFolder(object): for uid in uidlist: self.deletemessageflags(uid, flags) + def change_message_uid(self, uid, new_uid): + """Change the message from existing uid to new_uid + + If the backend supports it (IMAP does not). + :param new_uid: (optional) If given, the old UID will be changed + to a new UID. This allows backends efficient renaming of + messages if the UID has changed.""" + raise NotImplementedException + def deletemessage(self, uid): raise NotImplementedException @@ -275,20 +284,15 @@ class BaseFolder(object): #remained negative, no server was willing to assign us an #UID. If newid is 0, saving succeeded, but we could not #retrieve the new UID. Ignore message in this case. - newuid = dstfolder.savemessage(uid, message, flags, rtime) - - if newuid > 0: - if newuid != uid: + new_uid = dstfolder.savemessage(uid, message, flags, rtime) + if new_uid > 0: + if new_uid != uid: + # Got new UID, change the local uid to match the new one. + self.change_message_uid(uid, new_uid) + statusfolder.deletemessage(uid) # Got new UID, change the local uid. - #TODO: Maildir could do this with a rename rather than - #load/save/del operation, IMPLEMENT a changeuid() - #function or so. - self.savemessage(newuid, message, flags, rtime) - self.deletemessage(uid) - uid = newuid # Save uploaded status in the statusfolder - statusfolder.savemessage(uid, message, flags, rtime) - + statusfolder.savemessage(new_uid, message, flags, rtime) elif newuid == 0: # Message was stored to dstfolder, but we can't find it's UID # This means we can't link current message to the one created @@ -299,11 +303,11 @@ class BaseFolder(object): self.deletemessage(uid) else: raise OfflineImapError("Trying to save msg (uid %d) on folder " - "%s returned invalid uid %d" % \ - (uid, - dstfolder.getvisiblename(), - newuid), + "%s returned invalid uid %d" % (uid, + dstfolder.getvisiblename(), new_uid), OfflineImapError.ERROR.MESSAGE) + except (KeyboardInterrupt): # bubble up CTRL-C + raise except OfflineImapError, e: if e.severity > OfflineImapError.ERROR.MESSAGE: raise # buble severe errors up @@ -311,10 +315,9 @@ class BaseFolder(object): except Exception, e: self.ui.error(e, "Copying message %s [acc: %s]:\n %s" %\ (uid, self.accountname, - traceback.format_exc())) + exc_info()[2])) raise #raise on unknown errors, so we can fix those - def syncmessagesto_copy(self, dstfolder, statusfolder): """Pass1: Copy locally existing messages not on the other side diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 4471387..5d8d1c0 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -597,8 +597,8 @@ class IMAPFolder(BaseFolder): self.ui.debug('imap', 'savemessage: returning new UID %d' % uid) return uid - def savemessageflags(self, uid, flags): + """Change a message's flags to `flags`.""" imapobj = self.imapserver.acquireconnection() try: try: @@ -684,6 +684,14 @@ class IMAPFolder(BaseFolder): elif operation == '-': self.messagelist[uid]['flags'] -= flags + def change_message_uid(self, uid, new_uid): + """Change the message from existing uid to new_uid + + If the backend supports it. IMAP does not and will throw errors.""" + raise OfflineImapError('IMAP backend cannot change a messages UID from ' + '%d to %d' % (uid, new_uid), + OfflineImapError.ERROR.MESSAGE) + def deletemessage(self, uid): self.deletemessages_noconvert([uid]) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 5b2de91..22a5ef4 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -273,7 +273,7 @@ class MaildirFolder(BaseFolder): if rtime != None: os.utime(os.path.join(tmpdir, messagename), (rtime, rtime)) - self.messagelist[uid] = {'flags': set(), + self.messagelist[uid] = {'flags': flags, 'filename': os.path.join('tmp', messagename)} # savemessageflags moves msg to 'cur' or 'new' as appropriate self.savemessageflags(uid, flags) @@ -284,25 +284,25 @@ class MaildirFolder(BaseFolder): return self.messagelist[uid]['flags'] def savemessageflags(self, uid, flags): + """Sets the specified message's flags to the given set. + + This function moves the message to the cur or new subdir, + depending on the Seen flag.""" + # TODO: This function could be improved to only parse the + # filenames if the flags actually changed. oldfilename = self.messagelist[uid]['filename'] - dir_prefix, newname = os.path.split(oldfilename) - tmpdir = os.path.join(self.getfullname(), 'tmp') - if 'S' in flags: - # If a message has been seen, it goes into the cur - # directory. CR debian#152482 - dir_prefix = 'cur' - else: - dir_prefix = 'new' - - # Strip off existing infostring (preserving small letter flags, that + dir_prefix, filename = os.path.split(oldfilename) + # If a message has been seen, it goes into 'cur' + dir_prefix = 'cur' if 'S' in flags else 'new' + # Strip off existing infostring (preserving small letter flags that # dovecot uses) - infomatch = self.flagmatchre.search(newname) + infomatch = self.flagmatchre.search(filename) if infomatch: - newname = newname[:-len(infomatch.group())] #strip off + filename = filename[:-len(infomatch.group())] #strip off infostr = '%s2,%s' % (self.infosep, ''.join(sorted(flags))) - newname += infostr + filename += infostr + newfilename = os.path.join(dir_prefix, filename) - newfilename = os.path.join(dir_prefix, newname) if (newfilename != oldfilename): try: os.rename(os.path.join(self.getfullname(), oldfilename), @@ -315,10 +315,26 @@ class MaildirFolder(BaseFolder): self.messagelist[uid]['flags'] = flags self.messagelist[uid]['filename'] = newfilename - # By now, the message had better not be in tmp/ land! - final_dir, final_name = os.path.split(self.messagelist[uid]['filename']) - assert final_dir != 'tmp' + def change_message_uid(self, uid, new_uid): + """Change the message from existing uid to new_uid + This will not update the statusfolder UID, you need to do that yourself. + :param new_uid: (optional) If given, the old UID will be changed + to a new UID. The Maildir backend can implement this as an efficient + rename.""" + if not uid in self.messagelist: + raise OfflineImapError("Cannot change unknown Maildir UID %s" % uid) + if uid == new_uid: return + + oldfilename = self.messagelist[uid]['filename'] + dir_prefix, filename = os.path.split(oldfilename) + flags = self.getmessageflags(uid) + filename = self.new_message_filename(new_uid, flags) + os.rename(os.path.join(self.getfullname(), oldfilename), + os.path.join(self.getfullname(), dir_prefix, filename)) + self.messagelist[new_uid] = self.messagelist[uid] + del self.messagelist[uid] + def deletemessage(self, uid): """Unlinks a message file from the Maildir. diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 5da8e00..6b361f2 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -221,6 +221,31 @@ class MappedIMAPFolder(IMAPFolder): self._mb.addmessagesflags(self._uidlist(self.r2l, uidlist), flags) + def change_message_uid(self, ruid, new_ruid): + """Change the message from existing ruid to new_ruid + + :param new_uid: The old remote UID will be changed to a new + UID. The UIDMaps case handles this efficiently by simply + changing the mappings file.""" + if ruid not in self.r2l: + raise OfflineImapError("Cannot change unknown Maildir UID %s" % ruid, + OfflineImapError.ERROR.MESSAGE) + if ruid == new_ruid: return # sanity check shortcut + self.maplock.acquire() + try: + luid = self.r2l[ruid] + self.l2r[luid] = new_ruid + del self.r2l[ruid] + self.r2l[new_ruid] = luid + #TODO: diskl2r|r2l are a pain to sync and should be done away with + #diskl2r only contains positive UIDs, so wrap in ifs + if luid>0: self.diskl2r[luid] = new_ruid + if ruid>0: del self.diskr2l[ruid] + if new_ruid > 0: self.diskr2l[new_ruid] = luid + self._savemaps(dolock = 0) + finally: + self.maplock.release() + def _mapped_delete(self, uidlist): self.maplock.acquire() try: From ff50585007759b9dfe480ddbbf7a214010c3ce4f Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 30 Aug 2011 10:43:10 +0200 Subject: [PATCH 06/12] Folder-Maildir.savemessageflags(): Only parse file name if flags changed Rather than always parsing the filename, we only need to do so if the flags have actually changed, otherwise we can keep the filename. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Maildir.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index 22a5ef4..d7639f4 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -287,22 +287,24 @@ class MaildirFolder(BaseFolder): """Sets the specified message's flags to the given set. This function moves the message to the cur or new subdir, - depending on the Seen flag.""" - # TODO: This function could be improved to only parse the - # filenames if the flags actually changed. + depending on the 'S'een flag.""" + oldfilename = self.messagelist[uid]['filename'] dir_prefix, filename = os.path.split(oldfilename) # If a message has been seen, it goes into 'cur' dir_prefix = 'cur' if 'S' in flags else 'new' - # Strip off existing infostring (preserving small letter flags that - # dovecot uses) - infomatch = self.flagmatchre.search(filename) - if infomatch: - filename = filename[:-len(infomatch.group())] #strip off - infostr = '%s2,%s' % (self.infosep, ''.join(sorted(flags))) - filename += infostr - newfilename = os.path.join(dir_prefix, filename) + if flags != self.messagelist[uid]['flags']: + # Flags have actually changed, construct new filename + # Strip off existing infostring (preserving small letter flags that + # dovecot uses) + infomatch = self.flagmatchre.search(filename) + if infomatch: + filename = filename[:-len(infomatch.group())] #strip off + infostr = '%s2,%s' % (self.infosep, ''.join(sorted(flags))) + filename += infostr + + newfilename = os.path.join(dir_prefix, filename) if (newfilename != oldfilename): try: os.rename(os.path.join(self.getfullname(), oldfilename), From 78a37f27ef26f754eec56ae91cdaa3fc114a145c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 30 Sep 2011 17:20:11 +0200 Subject: [PATCH 07/12] WIP, revamp how we treat top-level dirs Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Base.py | 11 ++++++----- offlineimap/folder/IMAP.py | 2 +- offlineimap/folder/LocalStatus.py | 2 +- offlineimap/folder/Maildir.py | 2 +- offlineimap/repository/IMAP.py | 2 ++ 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 0e065e3..313728d 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -37,13 +37,14 @@ class BaseFolder(object): self.sync_this = True """Should this folder be included in syncing?""" self.ui = getglobalui() - self.name = name + # Top level dir name is always '' + self.name = name if not name == self.getsep() else '' self.repository = repository self.visiblename = repository.nametrans(name) - # In case the visiblename becomes '.' (top-level) we use '' as - # that is the name that e.g. the Maildir scanning will return - # for the top-level dir. - if self.visiblename == '.': + # In case the visiblename becomes '.' or '/' (top-level) we use + # '' as that is the name that e.g. the Maildir scanning will + # return for the top-level dir. + if self.visiblename == self.getsep(): self.visiblename = '' self.config = repository.getconfig() diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 5d8d1c0..d804766 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -33,10 +33,10 @@ except NameError: class IMAPFolder(BaseFolder): def __init__(self, imapserver, name, repository): name = imaputil.dequote(name) + self.sep = imapserver.delim super(IMAPFolder, self).__init__(name, repository) self.expunge = repository.getexpunge() self.root = None # imapserver.root - self.sep = imapserver.delim self.imapserver = imapserver self.messagelist = None self.randomgenerator = random.Random() diff --git a/offlineimap/folder/LocalStatus.py b/offlineimap/folder/LocalStatus.py index fd4228c..3d2cd37 100644 --- a/offlineimap/folder/LocalStatus.py +++ b/offlineimap/folder/LocalStatus.py @@ -27,8 +27,8 @@ magicline = "OFFLINEIMAP LocalStatus CACHE DATA - DO NOT MODIFY - FORMAT 1" class LocalStatusFolder(BaseFolder): def __init__(self, name, repository): + self.sep = '.' #needs to be set before super.__init__() super(LocalStatusFolder, self).__init__(name, repository) - self.sep = '.' self.filename = os.path.join(self.getroot(), self.getfolderbasename()) self.messagelist = {} self.savelock = threading.Lock() diff --git a/offlineimap/folder/Maildir.py b/offlineimap/folder/Maildir.py index d7639f4..973bc92 100644 --- a/offlineimap/folder/Maildir.py +++ b/offlineimap/folder/Maildir.py @@ -60,10 +60,10 @@ def gettimeseq(): class MaildirFolder(BaseFolder): def __init__(self, root, name, sep, repository): + self.sep = sep # needs to be set before super().__init__ super(MaildirFolder, self).__init__(name, repository) self.dofsync = self.config.getdefaultboolean("general", "fsync", True) self.root = root - self.sep = sep self.messagelist = None # check if we should use a different infosep to support Win file systems self.wincompatible = self.config.getdefaultboolean( diff --git a/offlineimap/repository/IMAP.py b/offlineimap/repository/IMAP.py index 8635f7c..a5ed84c 100644 --- a/offlineimap/repository/IMAP.py +++ b/offlineimap/repository/IMAP.py @@ -324,6 +324,8 @@ class IMAPRepository(BaseRepository): :param foldername: Full path of the folder to be created.""" if self.getreference(): foldername = self.getreference() + self.getsep() + foldername + if not foldername: # Create top level folder as folder separator + foldername = self.getsep() imapobj = self.imapserver.acquireconnection() try: From 00c67881a02200afb8dac9cec8149bf5ec5077d4 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 19:35:56 +0100 Subject: [PATCH 08/12] docs/MANUAL: How to upgrade to sqlite storage Add section to the manual describing how to upgrade to the new cache backend. Signed-off-by: Sebastian Spaeth --- docs/MANUAL.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/MANUAL.rst b/docs/MANUAL.rst index 49dedd6..da09141 100644 --- a/docs/MANUAL.rst +++ b/docs/MANUAL.rst @@ -305,6 +305,25 @@ achieve this. to hit the disk before continueing, you can set this to True. If you set it to False, you lose some of that safety, trading it for speed. + +Upgrading from plain text cache to SQLITE based cache +===================================================== + +OfflineImap uses a cache to store the last know status of mails (flags etc). Historically that has meant plain text files, but recently we introduced sqlite-based cache, which helps with performance and CPU usage on large folders. Here is how to upgrade existing plain text cache installations to sqlite based one: + + 1) Sync to make sure things are reasonably similar + 3) Change the account section to status_backend = sqlite + 4) A new sync will convert your plain text cache to an sqlite cache (but + leave the old plain text cache around for easy reverting) + This should be quick and not involve any mail up/downloading. + 5) See if it works :-) + 6a) If it does not work, go back to the old version or set + status_backend=plain + 6b) Or once you are sure it works, you can delete the + .offlineimap/Account-foo/LocalStatus folder (the new cache will be in + the LocalStatus-sqlite folder) + + Security and SSL ================ From 5240d3f36722c416f81c55774c7e7714c266aaf6 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 19:51:58 +0100 Subject: [PATCH 09/12] docs/INSTALL: How to uninstall Add a blurb on how to uninstall system wide installations. Signed-off-by: Sebastian Spaeth --- docs/INSTALL.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/INSTALL.rst b/docs/INSTALL.rst index 684ca6f..e2ed87a 100644 --- a/docs/INSTALL.rst +++ b/docs/INSTALL.rst @@ -110,3 +110,23 @@ within the sample file. `OfflineIMAP`_ also ships a file named `offlineimap.conf.minimal` that you can also try. It's useful if you want to get started with the most basic feature set, and you can read about other features later with `offlineimap.conf`. + + +=============== +Uninstall +=============== + +If you installed a system-wide installation via "python setup.py +install", there are a few files to purge to uninstall it again. I assume +that /usr/local is the standard prefix that your system and you use +python 2.7. Adapt to your system. In that case you need to: + +1) Delete: + /usr/local/lib/python2.7/dist-packages/offlineimap-6.4.4.egg-info + /usr/local/lib/python2.7/dist-packages/offlineimap + +2) Delete the cache at (default location) ~/.offlineimap + Delete your manually created (default loc) ~/.offlineimaprc + (It is possible that you created those in different spots) + +That's it. Have fun without OfflineImap. From 0ea6c6ed471b7de14dd42a37f4b80d99df60028e Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 21:41:04 +0100 Subject: [PATCH 10/12] Forgot to change a variable name in all cases the newuid var was renamed new_uid but there was one leftover. Fixed. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/Base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/Base.py b/offlineimap/folder/Base.py index 313728d..9ff60e2 100644 --- a/offlineimap/folder/Base.py +++ b/offlineimap/folder/Base.py @@ -294,7 +294,7 @@ class BaseFolder(object): # Got new UID, change the local uid. # Save uploaded status in the statusfolder statusfolder.savemessage(new_uid, message, flags, rtime) - elif newuid == 0: + elif new_uid == 0: # Message was stored to dstfolder, but we can't find it's UID # This means we can't link current message to the one created # in IMAP. So we just delete local message and on next run From 8fc72271895b7e46ad770417e2c5b8211f91eccd Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 21:57:48 +0100 Subject: [PATCH 11/12] SEVERE: Fix getting wrong UID back on IMAP upload This change looks harmless, but it fixes a severe bugfix, potentially leading to data loss! It fixes the "on n new uploads, it will redownload n-1, n-2, n-3,... messages during the next syncs" condition, and this is what happens: If there are more than one Mails to upload to a server, we do that by repeatedly invoking folder.IMAP.savemessage(). If the server supports the UIDPLUS extension we query the resulting UID by doing a: imapobj._get_untagged_response('APPENDUID', True) and that is exactly the problem. The "True" part causes the reply to remain in the "response stack" of the imaplib2 library. When we do the same call on a subsequent message and the connection is still on the same folder, we will get the same UID response back (imaplib2 only looks for the first matching response and returns that). The only time we clear the response stack, is when the IMAP connection SELECTS a different folder. This means that when we upload 10 messages, the IMAP server gives us always the same UID (that of the first one) back. And trying to write out 10 different messages with the same UID will confuse OfflineIMAP. This is the reason why we saw the ongoing UPLOADING/DOWNLOADING behavior that people reported. And this is the reason why we saw the inconsistency in the UID mapping in the IMAP<->IMAP case. I urge everyone to upgrade ASAP. Sorry for that, I don't know why the problem only became prevalent in the recent few releases as this code has been there for quite a while. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index d804766..68d1216 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -570,7 +570,7 @@ class IMAPFolder(BaseFolder): self.ui.warn("Server supports UIDPLUS but got no APPENDUID " "appending a message.") return 0 - uid = long(imapobj._get_untagged_response('APPENDUID', True)[-1].split(' ')[1]) + uid = long(imapobj._get_untagged_response('APPENDUID')[-1].split(' ')[1]) else: # we don't support UIDPLUS From d22390506882f68b523e482af4e781191533b3e6 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Fri, 6 Jan 2012 22:23:39 +0100 Subject: [PATCH 12/12] BUG FIX: Release v6.5.0 This is a CRITICAL bug fix release for everyone who is on the 6.4.x series. Please upgrade to avoid potential data loss! The version has been bumped to 6.5.0, please let everyone know to stay away from 6.5.x! I am sorry for this. See details in the Changelog and even more gory details in commit message for commit 8fc72271895b7e46ad770417e2c5b8211f91eccd. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 6 ------ Changelog.rst | 16 ++++++++++++++++ offlineimap/__init__.py | 2 +- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 5fd29e4..76a0ea3 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -16,11 +16,5 @@ New Features Changes ------- -* Uploading of Messages from Maildir and IMAP<->IMAP has been made more - efficient by renaming files/mapping entries, rather than actually - loading and saving the message under a new UID. - Bug Fixes --------- - -* Fix regression that broke MachineUI diff --git a/Changelog.rst b/Changelog.rst index 51bf175..21f9d42 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,22 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.5.0 (2012-01-06) +=============================== + +This is a CRITICAL bug fix release for everyone who is on the 6.4.x series. Please upgrade to avoid potential data loss! The version has been bumped to 6.5.0, please let everyone know that the 6.4.x series is problematic. + +* Uploading multiple emails to an IMAP server would lead to wrong UIDs + being returned (ie the same for all), which confused offlineimap and + led to recurrent upload/download loops and inconsistencies in the + IMAP<->IMAP uid mapping. + +* Uploading of Messages from Maildir and IMAP<->IMAP has been made more + efficient by renaming files/mapping entries, rather than actually + loading and saving the message under a new UID. + +* Fix regression that broke MachineUI + OfflineIMAP v6.4.4 (2012-01-06) =============================== diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 8406401..b16a9d6 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.4.4" +__version__ = "6.5.0" __copyright__ = "Copyright 2002-2012 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org"