From 0a275b9532a8de3be445dfc3b78e5d3bdb61199b Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 8 Jan 2012 01:06:56 +0100 Subject: [PATCH 1/8] Add scary warnings about "realdelete" option WARNING: I consider the Gmail "realdelete" option as harmful with the potential for DATALOSS. Add scary warnings to offlineimap.conf. See the analysis at http://article.gmane.org/gmane.mail.imap.offlineimap.general/5265 Deleting a message from a Gmail folder via the IMAP interface will just remove that folder's label from the message: the message will continue to exist in the '[Gmail]/All Mail' folder. If `realdelete` is set to `True`, then deleted messages will be moved to the '[Gmail]/Trash' folder. BEWARE: this will immediately delete a messages from *all folders* it belongs to! AS OFFLINEIMAP IMPLEMENTS FOLDER MOVES AS 1) AN ADD and 2) A DELETE (the order can vary), THIS MEANS THAT A FOLDER MOVE CAN CAUSE DATALOSS. DO NOT USE IT AND MOVE MAIL TO "[Gmail]/Trash" TO DELETE MAIL FROM "[Gmail]/All Mail"! We will need to discuss whether to completely disable that feature. Signed-off-by: Sebastian Spaeth --- offlineimap.conf | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/offlineimap.conf b/offlineimap.conf index 71ea2f7..c7f7d9d 100644 --- a/offlineimap.conf +++ b/offlineimap.conf @@ -546,16 +546,20 @@ type = Gmail # Specify the Gmail user name. This is the only mandatory parameter. remoteuser = username@gmail.com -# Deleting a message from a Gmail folder via the IMAP interface will -# just remove that folder's label from the message: the message will -# continue to exist in the '[Gmail]/All Mail' folder. If `realdelete` -# is set to `True`, then deleted messages will really be deleted -# during `offlineimap` sync, by moving them to the '[Gmail]/Trash' -# folder. BEWARE: this will delete a messages from *all folders* it -# belongs to! -# -# See http://mail.google.com/support/bin/answer.py?answer=77657&topic=12815 -realdelete = no +# WARNING: READ THIS BEFORE CONSIDERING TO CHANGE IT! Deleting a +# message from a Gmail folder via the IMAP interface will just remove +# that folder's label from the message: the message will continue to +# exist in the '[Gmail]/All Mail' folder. If `realdelete` is set to +# `True`, then deleted messages will be moved to the '[Gmail]/Trash' +# folder. BEWARE: this will immediately delete a messages from *all +# folders* it belongs to! AS OFFLINEIMAP IMPLEMENTS FOLDER MOVES AS 1) +# AN ADD and 2) A DELETE (the order can vary), THIS MEANS THAT A FOLDER +# MOVE CAN CAUSE DATALOSS. DO NOT USE IT AND MOVE MAIL TO +# "[Gmail]/Trash" TO DELETE MAIL FROM "[Gmail]/All Mail"! See the +# analysis at +# http://article.gmane.org/gmane.mail.imap.offlineimap.general/5265 See +# http://mail.google.com/support/bin/answer.py?answer=77657&topic=12815 +# realdelete = no !!!READ ABOVE BEFORE USING # The trash folder name may be different from [Gmail]/Trash # for example on german googlemail, this setting should be From ed718054764ab75c99abcfe37e291dafa4fd8118 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 8 Jan 2012 01:13:08 +0100 Subject: [PATCH 2/8] Changelog entry about "realdelete" option Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 76a0ea3..a9a73c7 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -10,6 +10,9 @@ others. `WIP (coming releases)` ======================= +* Gmail "realdelete" is considered harmful and has the potential for data loss. Analysis at http://article.gmane.org/gmane.mail.imap.offlineimap.general/5265 +Warnings were added to offlineimap.conf + New Features ------------ From 50de2174bfbf35dfcc1b0f9253e46218fb29b986 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 8 Jan 2012 11:29:54 +0100 Subject: [PATCH 3/8] Allow to pass 'force' arg to selectro() to enforce a new select Pass through the 'force' argument from selectro() to select() so that it can also enforce a new SELECT even if we already are on that folder. Also change the default parameter from '0' to 'False' to make clear that this is a Bool. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 8 ++++---- offlineimap/imaplibutil.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index 93f0c56..d23fb5f 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -42,19 +42,19 @@ class IMAPFolder(BaseFolder): self.randomgenerator = random.Random() #self.ui is set in BaseFolder - def selectro(self, imapobj): + def selectro(self, imapobj, force = False): """Select this folder when we do not need write access. Prefer SELECT to EXAMINE if we can, since some servers (Courier) do not stabilize UID validity until the folder is selected. .. todo: Still valid? Needs verification - + :param: Enforce new SELECT even if we are on that folder already. :returns: raises :exc:`OfflineImapError` severity FOLDER on error""" try: - imapobj.select(self.getfullname()) + imapobj.select(self.getfullname(), force = force) except imapobj.readonly: - imapobj.select(self.getfullname(), readonly = True) + imapobj.select(self.getfullname(), readonly = True, force = force) def suggeststhreads(self): return 1 diff --git a/offlineimap/imaplibutil.py b/offlineimap/imaplibutil.py index c9a7715..b4345fa 100644 --- a/offlineimap/imaplibutil.py +++ b/offlineimap/imaplibutil.py @@ -40,7 +40,7 @@ class UsefulIMAPMixIn(object): return self.mailbox return None - def select(self, mailbox='INBOX', readonly=False, force = 0): + def select(self, mailbox='INBOX', readonly=False, force = False): """Selects a mailbox on the IMAP server :returns: 'OK' on success, nothing if the folder was already From 7184ec28cced142c655c06d360250c7fb330809c Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 8 Jan 2012 12:26:47 +0100 Subject: [PATCH 4/8] Sanity check return value of UIDVALIDTY response We have a reported case where response('UIDVALIDITY') returned [None] which results in an ugly non-intuitive crash. Sanity check and report something nicer. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 1 + 1 file changed, 1 insertion(+) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index d23fb5f..f04f871 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -71,6 +71,7 @@ class IMAPFolder(BaseFolder): # SELECT receives UIDVALIDITY response self.selectro(imapobj) typ, uidval = imapobj.response('UIDVALIDITY') + assert uidval != [None], "response('UIDVALIDITY') returned [None]!" return long(uidval[0]) finally: self.imapserver.releaseconnection(imapobj) From 81f194adca78da848f7c139a526945358750ac32 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Sun, 8 Jan 2012 12:48:21 +0100 Subject: [PATCH 5/8] mbnames should write out local and not nametransformed box names Rather than to write out the nametrans'lated folder names for mbnames, we now write out the local untransformed box names. This is generally what we want. This became relevant since we support nametrans rules on the local side since only a short time. Reported by Paul Collignan. Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 5 +++++ offlineimap/accounts.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index a9a73c7..41d3af8 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -19,5 +19,10 @@ New Features Changes ------- +* Rather than to write out the nametrans'lated folder names for mbnames, + we now write out the local untransformed box names. This is generally + what we want. This became relevant since we support nametrans rules on + the local side since only a short time. Reported by Paul Collignan. + Bug Fixes --------- diff --git a/offlineimap/accounts.py b/offlineimap/accounts.py index 4644604..fc2687f 100644 --- a/offlineimap/accounts.py +++ b/offlineimap/accounts.py @@ -372,7 +372,7 @@ def syncfolder(account, remotefolder, quick): % localfolder) return # Write the mailboxes - mbnames.add(account.name, localfolder.getvisiblename()) + mbnames.add(account.name, localfolder.getname()) # Load status folder. statusfolder = statusrepos.getfolder(remotefolder.getvisiblename().\ From 3284e010ff12a8c6a80e8ea299af26e80edcd4c0 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 9 Jan 2012 09:51:43 +0100 Subject: [PATCH 6/8] Revert "use .response() rather _get_untagged_response()" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recently the internal function use of imaplib2's _get_untagged_response() was switched to use the public documented .response() function (which should return the same data). However within a few fays we received reports that both uses of a) the UIDVALIDITY fetching and b) the APPENDUID fetching returned [None] as data although the IMAP log definitely shows that data was returned. Revert to using the undocumented internal imaplib2 function, that seemed to have worked without problems. This needs to be taken up to the imaplib2 developer. Signed-off-by: Sebastian Spaeth --- offlineimap/folder/IMAP.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/offlineimap/folder/IMAP.py b/offlineimap/folder/IMAP.py index f04f871..75c731c 100644 --- a/offlineimap/folder/IMAP.py +++ b/offlineimap/folder/IMAP.py @@ -70,9 +70,12 @@ class IMAPFolder(BaseFolder): try: # SELECT receives UIDVALIDITY response self.selectro(imapobj) - typ, uidval = imapobj.response('UIDVALIDITY') + # note: we would want to use .response() here but that + # often seems to return [None], even though we have + # data. TODO + uidval = imapobj._get_untagged_response('UIDVALIDITY') assert uidval != [None], "response('UIDVALIDITY') returned [None]!" - return long(uidval[0]) + return long(uidval[-1]) finally: self.imapserver.releaseconnection(imapobj) @@ -567,8 +570,11 @@ class IMAPFolder(BaseFolder): if use_uidplus or imapobj._get_untagged_response('APPENDUID', True): # get new UID from the APPENDUID response, it could look # like OK [APPENDUID 38505 3955] APPEND completed with - # 38505 bein folder UIDvalidity and 3955 the new UID - typ, resp = imapobj.response('APPENDUID') + # 38505 bein folder UIDvalidity and 3955 the new UID. + # note: we would want to use .response() here but that + # often seems to return [None], even though we have + # data. TODO + resp = imapobj._get_untagged_response('APPENDUID') if resp == [None]: self.ui.warn("Server supports UIDPLUS but got no APPENDUID " "appending a message.") From d72bb88729b95d1a6b8e377c286680fbd24b04c8 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 9 Jan 2012 09:57:36 +0100 Subject: [PATCH 7/8] Improve error message Add *what* UID was returned in case savemessage did not return a UID>0 Signed-off-by: Sebastian Spaeth --- offlineimap/folder/UIDMaps.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/offlineimap/folder/UIDMaps.py b/offlineimap/folder/UIDMaps.py index 1a6283c..ac0a9b4 100644 --- a/offlineimap/folder/UIDMaps.py +++ b/offlineimap/folder/UIDMaps.py @@ -191,7 +191,8 @@ class MappedIMAPFolder(IMAPFolder): newluid = self._mb.savemessage(-1, content, flags, rtime) if newluid < 1: - raise ValueError("Backend could not find uid for message") + raise ValueError("Backend could not find uid for message, returned " + "%s" % newluid) self.maplock.acquire() try: self.diskl2r[newluid] = uid From 9e65cfca2161c023d9772924647533841cf0cc89 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Mon, 9 Jan 2012 10:04:05 +0100 Subject: [PATCH 8/8] Release v6.5.2-rc1 OfflineIMAP v6.5.2-rc1 (2012-01-09) =================================== Commits v6.5.1.1 - v6.5.2-rc1: note: Proper Changelog still in Changelog-draft.rst d72bb88 Improve error message 3284e01 Revert "use .response() rather _get_untagged_response()" 81f194a mbnames should write out local and not nametransformed box names 7184ec2 Sanity check return value of UIDVALIDTY response 50de217 Allow to pass 'force' arg to selectro() to enforce a new select ed71805 Changelog entry about "realdelete" option 0a275b9 Add scary warnings about "realdelete" option Signed-off-by: Sebastian Spaeth --- Changelog.draft.rst | 5 +++++ Changelog.rst | 12 ++++++++++++ offlineimap/__init__.py | 2 +- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Changelog.draft.rst b/Changelog.draft.rst index 41d3af8..bf0b460 100644 --- a/Changelog.draft.rst +++ b/Changelog.draft.rst @@ -24,5 +24,10 @@ Changes what we want. This became relevant since we support nametrans rules on the local side since only a short time. Reported by Paul Collignan. +* Some sanity checks and improved error messages. + +* Revert 6.5.1.1 change to use public imaplib2 function, it was reported to + not always work. + Bug Fixes --------- diff --git a/Changelog.rst b/Changelog.rst index 59550b9..f7a7dd3 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -11,6 +11,18 @@ ChangeLog on releases. And because I'm lazy, it will also be used as a draft for the releases announces. +OfflineIMAP v6.5.2-rc1 (2012-01-09) +=================================== +Commits v6.5.1.1 - v6.5.2-rc1: +note: Proper Changelog still in Changelog-draft.rst +d72bb88 Improve error message +3284e01 Revert "use .response() rather _get_untagged_response()" +81f194a mbnames should write out local and not nametransformed box names +7184ec2 Sanity check return value of UIDVALIDTY response +50de217 Allow to pass 'force' arg to selectro() to enforce a new select +ed71805 Changelog entry about "realdelete" option +0a275b9 Add scary warnings about "realdelete" option + OfflineIMAP v6.5.1.2 (2012-01-07) - "Baby steps" ================================================ diff --git a/offlineimap/__init__.py b/offlineimap/__init__.py index 2bf4637..99fc2b0 100644 --- a/offlineimap/__init__.py +++ b/offlineimap/__init__.py @@ -1,7 +1,7 @@ __all__ = ['OfflineImap'] __productname__ = 'OfflineIMAP' -__version__ = "6.5.1.2" +__version__ = "6.5.2-rc1" __copyright__ = "Copyright 2002-2012 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "john@complete.org"